diff --git a/src/promptflow-evals/promptflow/evals/evaluate/_eval_run.py b/src/promptflow-evals/promptflow/evals/evaluate/_eval_run.py new file mode 100644 index 00000000000..a456fd45258 --- /dev/null +++ b/src/promptflow-evals/promptflow/evals/evaluate/_eval_run.py @@ -0,0 +1,394 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- +from typing import Any, Dict, Optional, Type + +import dataclasses +import json +import logging +import os +import posixpath +import requests +import uuid + +from azure.ai.ml import MLClient +from azure.storage.blob import BlobClient +from requests.adapters import HTTPAdapter +from urllib.parse import urlparse +from urllib3.util.retry import Retry + +from promptflow.evals._version import VERSION +import time + +LOGGER = logging.getLogger(__name__) + + +@dataclasses.dataclass +class RunInfo(): + """ + A holder for run info, needed for logging. + """ + run_id: str + experiment_id: str + + @staticmethod + def generate() -> 'RunInfo': + """ + Generate the new RunInfo instance with the RunID and Experiment ID. + """ + return RunInfo( + str(uuid.uuid4()), + str(uuid.uuid4()), + ) + + +class Singleton(type): + """Singleton class, which will be used as a metaclass.""" + _instances = {} + + def __call__(cls, *args, **kwargs): + """Redefinition of call to return one instance per type.""" + if cls not in Singleton._instances: + Singleton._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + return Singleton._instances[cls] + + @staticmethod + def destroy(cls: Type) -> None: + """ + Destroy the singleton instance. + + :param cls: The class to be destroyed. + """ + Singleton._instances.pop(cls, None) + + +class EvalRun(metaclass=Singleton): + ''' + The simple singleton run class, used for accessing artifact store. + + :param run_name: The name of the run. + :type run_name: Optional[str] + :param tracking_uri: Tracking URI for this run; required to make calls. + :type tracking_uri: str + :param subscription_id: The subscription ID used to track run. + :type subscription_id: str + :param group_name: The resource group used to track run. + :type group_name: str + :param workspace_name: The name of workspace/project used to track run. + :type workspace_name: str + :param ml_client: The ml client used for authentication into Azure. + :type ml_client: MLClient + ''' + + _MAX_RETRIES = 5 + _BACKOFF_FACTOR = 2 + _TIMEOUT = 5 + _SCOPE = "https://management.azure.com/.default" + + def __init__(self, + run_name: Optional[str], + tracking_uri: str, + subscription_id: str, + group_name: str, + workspace_name: str, + ml_client: MLClient + ): + """ + Constructor + """ + + self._tracking_uri: str = tracking_uri + self._subscription_id: str = subscription_id + self._resource_group_name: str = group_name + self._workspace_name: str = workspace_name + self._ml_client: MLClient = ml_client + self._url_base = urlparse(self._tracking_uri).netloc + self._is_broken = self._start_run() + self._is_terminated = False + self.name: str = run_name if run_name else self.info.run_id + + def _get_scope(self): + """ + Return the scope information for the workspace. + + :param workspace_object: The workspace object. + :type workspace_object: azureml.core.workspace.Workspace + :return: The scope information for the workspace. + :rtype: str + """ + return ( + "/subscriptions/{}/resourceGroups/{}/providers" + "/Microsoft.MachineLearningServices" + "/workspaces/{}" + ).format( + self._subscription_id, + self._resource_group_name, + self._workspace_name, + ) + + def _start_run(self) -> bool: + """ + Make a request to start the mlflow run. If the run will not start, it will be + + marked as broken and the logging will be switched off. + :returns: True if the run has started and False otherwise. + """ + url = ( + f"https://{self._url_base}/mlflow/v2.0" + f"{self._get_scope()}/api/2.0/mlflow/runs/create") + body = { + "experiment_id": "0", + "user_id": "promptflow-evals", + "start_time": int(time.time() * 1000), + "tags": [ + { + "key": "mlflow.user", + "value": "promptflow-evals" + } + ] + } + response = self.request_with_retry( + url=url, + method='POST', + json_dict=body + ) + if response.status_code != 200: + self.info = RunInfo.generate() + LOGGER.error(f"The run failed to start: {response.status_code}: {response.text}." + "The results will be saved locally, but will not be logged to Azure.") + return True + parsed_response = response.json() + self.info = RunInfo( + run_id=parsed_response['run']['info']['run_id'], + experiment_id=parsed_response['run']['info']['experiment_id'], + ) + return False + + def end_run(self, status: str) -> None: + """ + Tetminate the run. + + :param status: One of "FINISHED" "FAILED" and "KILLED" + :type status: str + :raises: ValueError if the run is not in ("FINISHED", "FAILED", "KILLED") + """ + if status not in ("FINISHED", "FAILED", "KILLED"): + raise ValueError( + f"Incorrect terminal status {status}. " + "Valid statuses are \"FINISHED\", \"FAILED\" and \"KILLED\".") + if self._is_terminated: + LOGGER.warning("Unable to stop run because it was already terminated.") + return + if self._is_broken: + LOGGER.error("Unable to stop run because the run failed to start.") + return + url = ( + f"https://{self._url_base}/mlflow/v2.0" + f"{self._get_scope()}/api/2.0/mlflow/runs/update") + body = { + "run_uuid": self.info.run_id, + "status": status, + "end_time": int(time.time() * 1000), + "run_id": self.info.run_id + } + response = self.request_with_retry( + url=url, + method='POST', + json_dict=body + ) + if response.status_code != 200: + LOGGER.error("Unable to terminate the run.") + Singleton.destroy(EvalRun) + self._is_terminated = True + + def get_run_history_uri(self) -> str: + """ + Return the run history service URI. + """ + return ( + f"https://{self._url_base}" + "/history/v1.0" + f"{self._get_scope()}" + f'/experimentids/{self.info.experiment_id}/runs/{self.info.run_id}' + ) + + def get_artifacts_uri(self) -> str: + """ + Returns the url to upload the artifacts. + """ + return self.get_run_history_uri() + '/artifacts/batch/metadata' + + def get_metrics_url(self): + """ + Return the url needed to track the mlflow metrics. + """ + return ( + f"https://{self._url_base}" + "/mlflow/v2.0" + f"{self._get_scope()}" + f'/api/2.0/mlflow/runs/log-metric' + ) + + def _get_token(self): + """The simple method to get token from the MLClient.""" + # This behavior mimics how the authority is taken in azureml-mlflow. + # Note, that here we are taking authority for public cloud, however, + # it will not work for non-public clouds. + return self._ml_client._credential.get_token(EvalRun._SCOPE) + + def request_with_retry( + self, + url: str, + method: str, + json_dict: Dict[str, Any], + headers: Optional[Dict[str, str]] = None + ) -> requests.Response: + """ + Send the request with retries. + + :param url: The url to send the request to. + :type url: str + :param auth_token: Azure authentication token + :type auth_token: str or None + :param method: The request method to be used. + :type method: str + :param json_dict: The json dictionary (not serialized) to be sent. + :type json_dict: dict. + :return: The requests.Response object. + """ + if headers is None: + headers = {} + headers['User-Agent'] = f'promptflow/{VERSION}' + headers['Authorization'] = f'Bearer {self._get_token().token}' + retry = Retry( + total=EvalRun._MAX_RETRIES, + connect=EvalRun._MAX_RETRIES, + read=EvalRun._MAX_RETRIES, + redirect=EvalRun._MAX_RETRIES, + status=EvalRun._MAX_RETRIES, + status_forcelist=(408, 429, 500, 502, 503, 504), + backoff_factor=EvalRun._BACKOFF_FACTOR, + allowed_methods=None + ) + adapter = HTTPAdapter(max_retries=retry) + session = requests.Session() + session.mount("https://", adapter) + return session.request( + method, + url, + headers=headers, + json=json_dict, + timeout=EvalRun._TIMEOUT + ) + + def _log_error(self, failed_op: str, response: requests.Response) -> None: + """ + Log the error if request was not successful. + + :param failed_op: The user-friendly message for the failed operation. + :type failed_op: str + :param response: The request. + :type response: requests.Response + """ + LOGGER.error( + f"Unable to {failed_op}, " + f"the request failed with status code {response.status_code}, " + f"{response.text=}." + ) + + def log_artifact(self, artifact_folder: str) -> None: + """ + The local implementation of mlflow-like artifact logging. + + **Note:** In the current implementation we are not using the thread pool executor + as it is done in azureml-mlflow, instead we are just running upload in cycle as we are not + expecting uploading a lot of artifacts. + :param artifact_folder: The folder with artifacts to be uploaded. + :type artifact_folder: str + """ + if self._is_broken: + LOGGER.error("Unable to log artifact because the run failed to start.") + return + # Check if artifact dirrectory is empty or does not exist. + if not os.path.isdir(artifact_folder): + LOGGER.error("The path to the artifact is either not a directory or does not exist.") + return + if not os.listdir(artifact_folder): + LOGGER.error("The path to the artifact is empty.") + return + # First we will list the files and the appropriate remote paths for them. + upload_path = os.path.basename(os.path.normpath(artifact_folder)) + remote_paths = {'paths': []} + local_paths = [] + + for (root, _, filenames) in os.walk(artifact_folder): + if root != artifact_folder: + rel_path = os.path.relpath(root, artifact_folder) + if rel_path != '.': + upload_path = posixpath.join(upload_path, rel_path) + for f in filenames: + remote_file_path = posixpath.join(upload_path, f) + remote_paths['paths'].append({'path': remote_file_path}) + local_file_path = os.path.join(root, f) + local_paths.append(local_file_path) + # Now we need to reserve the space for files in the artifact store. + headers = { + 'Content-Type': "application/json", + 'Accept': "application/json", + 'Content-Length': str(len(json.dumps(remote_paths))), + 'x-ms-client-request-id': str(uuid.uuid1()), + } + response = self.request_with_retry( + url=self.get_artifacts_uri(), + method='POST', + json_dict=remote_paths, + headers=headers + ) + if response.status_code != 200: + self._log_error("allocate Blob for the artifact", response) + return + empty_artifacts = response.json()['artifactContentInformation'] + # The response from Azure contains the URL with SAS, that allows to upload file to the + # artifact store. + for local, remote in zip(local_paths, remote_paths['paths']): + artifact_loc = empty_artifacts[remote['path']] + blob_client = BlobClient.from_blob_url(artifact_loc['contentUri'], max_single_put_size=32 * 1024 * 1024) + with open(local, 'rb') as fp: + blob_client.upload_blob(fp) + + def log_metric(self, key: str, value: float) -> None: + """ + Log the metric to azure similar to how it is done by mlflow. + + :param key: The metric name to be logged. + :type key: str + :param value: The valure to be logged. + :type value: float + """ + if self._is_broken: + LOGGER.error("Unable to log metric because the run failed to start.") + return + body = { + "run_uuid": self.info.run_id, + "key": key, + "value": value, + "timestamp": int(time.time() * 1000), + "step": 0, + "run_id": self.info.run_id + } + response = self.request_with_retry( + url=self.get_metrics_url(), + method='POST', + json_dict=body, + ) + if response.status_code != 200: + self._log_error('save metrics', response) + + @staticmethod + def get_instance(*args, **kwargs) -> "EvalRun": + """ + The convenience method to the the EvalRun instance. + + :return: The EvalRun instance. + """ + return EvalRun(*args, **kwargs) diff --git a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py index 07e2f0b9644..6fae014101d 100644 --- a/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py +++ b/src/promptflow-evals/promptflow/evals/evaluate/_evaluate.py @@ -42,10 +42,10 @@ def _aggregate_metrics(df, evaluators) -> Dict[str, float]: # Check the namespace of the evaluator module = inspect.getmodule(evaluators[evaluator_name]) if ( - module - and module.__name__.startswith("promptflow.evals.evaluators.") - and metric_name.endswith("_score") - and metric_name.replace("_score", "") in content_safety_metrics + module and + module.__name__.startswith("promptflow.evals.evaluators.") and + metric_name.endswith("_score") and + metric_name.replace("_score", "") in content_safety_metrics ): content_safety_cols.append(col) @@ -55,8 +55,8 @@ def _aggregate_metrics(df, evaluators) -> Dict[str, float]: defect_rate_name = col.replace("_score", "_defect_rate") col_with_numeric_values = pd.to_numeric(content_safety_df[col], errors='coerce') defect_rates[defect_rate_name] = round( - np.sum(col_with_numeric_values >= CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT) - / col_with_numeric_values.count(), + np.sum(col_with_numeric_values >= CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT) / + col_with_numeric_values.count(), 2, ) @@ -115,7 +115,8 @@ def _validate_and_load_data(target, data, evaluators, output_path, azure_ai_proj try: initial_data_df = pd.read_json(data, lines=True) except Exception as e: - raise ValueError(f"Failed to load data from {data}. Please validate it is a valid jsonl data. Error: {str(e)}.") + raise ValueError( + f"Failed to load data from {data}. Please validate it is a valid jsonl data. Error: {str(e)}.") return initial_data_df @@ -154,9 +155,13 @@ def _validate_columns( _validate_input_data_for_evaluator(evaluator, evaluator_name, new_df) -def _apply_target_to_data( - target: Callable, data: str, pf_client: PFClient, initial_data: pd.DataFrame, evaluation_name: Optional[str] = None -) -> Tuple[pd.DataFrame, Set[str]]: +def _apply_target_to_data(target: Callable, + data: str, + pf_client: PFClient, + initial_data: pd.DataFrame, + evaluation_name: Optional[str] = None, + _run_name: Optional[str] = None) -> Tuple[pd.DataFrame, + Set[str]]: """ Apply the target function to the data set and return updated data and generated columns. @@ -168,6 +173,8 @@ def _apply_target_to_data( :paramtype pf_client: PFClient :keyword initial_data: The data frame with the loaded data. :paramtype initial_data: pd.DataFrame + :keyword _run_name: The name of target run. Used for testing only. + :paramtype _run_name: Optional[str] :return: The tuple, containing data frame and the list of added columns. :rtype: Tuple[pd.DataFrame, List[str]] """ @@ -180,11 +187,12 @@ def _apply_target_to_data( data=data, properties={"runType": "eval_run", "isEvaluatorRun": "true"}, stream=True, + name=_run_name ) target_output = pf_client.runs.get_details(run, all_results=True) # Remove input and output prefix generated_columns = { - col[len(Prefixes._OUTPUTS) :] for col in target_output.columns if col.startswith(Prefixes._OUTPUTS) + col[len(Prefixes._OUTPUTS):] for col in target_output.columns if col.startswith(Prefixes._OUTPUTS) } # Sort output by line numbers target_output.set_index(f"inputs.{LINE_NUMBER}", inplace=True) @@ -350,7 +358,8 @@ def evaluate( target_generated_columns = set() if data is not None and target is not None: input_data_df, target_generated_columns, target_run = _apply_target_to_data( - target, data, pf_client, input_data_df, evaluation_name + target, data, pf_client, input_data_df, evaluation_name, + _run_name=kwargs.get('_run_name') ) # Make sure, the default is always in the configuration. @@ -431,7 +440,7 @@ def evaluate( metrics = _aggregate_metrics(evaluators_result_df, evaluators) studio_url = _log_metrics_and_instance_results( - metrics, result_df, trace_destination, target_run, pf_client, data, evaluation_name + metrics, result_df, trace_destination, target_run ) result = {"rows": result_df.to_dict("records"), "metrics": metrics, "studio_url": studio_url} diff --git a/src/promptflow-evals/promptflow/evals/evaluate/_utils.py b/src/promptflow-evals/promptflow/evals/evaluate/_utils.py index 2eabe4d3502..bd228b0581f 100644 --- a/src/promptflow-evals/promptflow/evals/evaluate/_utils.py +++ b/src/promptflow-evals/promptflow/evals/evaluate/_utils.py @@ -9,13 +9,11 @@ from collections import namedtuple from pathlib import Path -import mlflow import pandas as pd -from promptflow._sdk._constants import Local2Cloud -from promptflow._utils.async_utils import async_run_allowing_running_loop -from promptflow.azure._dependencies._pf_evals import AsyncRunUploader from promptflow.evals._constants import DEFAULT_EVALUATION_RESULTS_FILE_NAME, Prefixes +from promptflow.evals.evaluate._eval_run import EvalRun + LOGGER = logging.getLogger(__name__) @@ -51,24 +49,13 @@ def load_jsonl(path): def _write_properties_to_run_history(properties: dict) -> None: - from mlflow.tracking import MlflowClient - from mlflow.utils.rest_utils import http_request - - # get mlflow run - run = mlflow.active_run() - if run is None: - run = mlflow.start_run() - # get auth from client - client = MlflowClient() + run = EvalRun.get_instance() try: - cred = client._tracking_client.store.get_host_creds() # pylint: disable=protected-access # update host to run history and request PATCH API - cred.host = cred.host.replace("mlflow/v2.0", "mlflow/v1.0").replace("mlflow/v1.0", "history/v1.0") - response = http_request( - host_creds=cred, - endpoint=f"/experimentids/{run.info.experiment_id}/runs/{run.info.run_id}", + response = run.request_with_retry( + url=run.get_run_history_uri(), method="PATCH", - json={"runId": run.info.run_id, "properties": properties}, + json_dict={"runId": run.info.run_id, "properties": properties}, ) if response.status_code != 200: LOGGER.error("Fail writing properties '%s' to run history: %s", properties, response.text) @@ -77,7 +64,7 @@ def _write_properties_to_run_history(properties: dict) -> None: LOGGER.error("Fail writing properties '%s' to run history: %s", properties, e) -def _azure_pf_client(trace_destination): +def _azure_pf_client_and_triad(trace_destination): from promptflow.azure._cli._utils import _get_azure_pf_client ws_triad = extract_workspace_triad_from_trace_provider(trace_destination) @@ -87,88 +74,56 @@ def _azure_pf_client(trace_destination): workspace_name=ws_triad.workspace_name, ) - return azure_pf_client - - -def _get_mlflow_tracking_uri(trace_destination): - azure_pf_client = _azure_pf_client(trace_destination) - ws_triad = extract_workspace_triad_from_trace_provider(trace_destination) - - ws = azure_pf_client.ml_client.workspaces.get(ws_triad.workspace_name) - return ws.mlflow_tracking_uri - - -def _get_trace_destination_config(tracking_uri): - from promptflow._sdk._configuration import Configuration - - pf_config = Configuration(overrides={"trace.destination": tracking_uri} if tracking_uri is not None else None) - - trace_destination = pf_config.get_trace_destination() - - if is_none(trace_destination): - return None - - return trace_destination + return azure_pf_client, ws_triad def _log_metrics_and_instance_results( - metrics, instance_results, tracking_uri, run, pf_client, data, evaluation_name=None + metrics, instance_results, trace_destination, run ) -> str: - run_id = None - trace_destination = _get_trace_destination_config(tracking_uri=tracking_uri) - if trace_destination is None: + LOGGER.error("Unable to log traces as trace destination was not defined.") return None - tracking_uri = _get_mlflow_tracking_uri(trace_destination=trace_destination) + azure_pf_client, ws_triad = _azure_pf_client_and_triad(trace_destination) + tracking_uri = azure_pf_client.ml_client.workspaces.get(ws_triad.workspace_name).mlflow_tracking_uri # Adding line_number as index column this is needed by UI to form link to individual instance run instance_results["line_number"] = instance_results.index - if run is None: - mlflow.set_tracking_uri(tracking_uri) - - with tempfile.TemporaryDirectory() as tmpdir: - with mlflow.start_run(run_name=evaluation_name) as run: - tmp_path = os.path.join(tmpdir, "eval_results.jsonl") - - with open(tmp_path, "w", encoding="utf-8") as f: - f.write(instance_results.to_json(orient="records", lines=True)) - - mlflow.log_artifact(tmp_path) - - # Using mlflow to create a dummy run since once created via PF show traces of dummy run in UI. - # Those traces can be confusing. - # adding these properties to avoid showing traces if a dummy run is created - _write_properties_to_run_history( - properties={ - "_azureml.evaluation_run": "azure-ai-generative-parent", - "_azureml.evaluate_artifacts": json.dumps([{"path": "eval_results.jsonl", "type": "table"}]), - "isEvaluatorRun": "true", - } - ) - run_id = run.info.run_id - else: - azure_pf_client = _azure_pf_client(trace_destination=trace_destination) - with tempfile.TemporaryDirectory() as temp_dir: - file_name = Local2Cloud.FLOW_INSTANCE_RESULTS_FILE_NAME - local_file = Path(temp_dir) / file_name - instance_results.to_json(local_file, orient="records", lines=True) - - # overriding instance_results.jsonl file - async_uploader = AsyncRunUploader._from_run_operations(azure_pf_client.runs) - remote_file = ( - f"{Local2Cloud.BLOB_ROOT_PROMPTFLOW}" - f"/{Local2Cloud.BLOB_ARTIFACTS}/{run.name}/{Local2Cloud.FLOW_INSTANCE_RESULTS_FILE_NAME}" - ) - async_run_allowing_running_loop(async_uploader._upload_local_file_to_blob, local_file, remote_file) - run_id = run.name - - client = mlflow.tracking.MlflowClient(tracking_uri=tracking_uri) + ev_run = EvalRun( + run_name=run.name if run is not None else None, + tracking_uri=tracking_uri, + subscription_id=ws_triad.subscription_id, + group_name=ws_triad.resource_group_name, + workspace_name=ws_triad.workspace_name, + ml_client=azure_pf_client.ml_client + ) + with tempfile.TemporaryDirectory() as tmpdir: + eval_path = os.path.join(tmpdir, "evaluation_results") + os.makedirs(eval_path, exist_ok=True) + tmp_path = os.path.join(eval_path, "eval_results.jsonl") + + with open(tmp_path, "w", encoding="utf-8") as f: + f.write(instance_results.to_json(orient="records", lines=True)) + + ev_run.log_artifact(eval_path) + + # Using mlflow to create a dummy run since once created via PF show traces of dummy run in UI. + # Those traces can be confusing. + # adding these properties to avoid showing traces if a dummy run is created + _write_properties_to_run_history( + properties={ + "_azureml.evaluation_run": "azure-ai-generative-parent", + "_azureml.evaluate_artifacts": json.dumps([{"path": "eval_results.jsonl", "type": "table"}]), + "isEvaluatorRun": "true", + } + ) + for metric_name, metric_value in metrics.items(): - client.log_metric(run_id, metric_name, metric_value) + ev_run.log_metric(metric_name, metric_value) - return _get_ai_studio_url(trace_destination=trace_destination, evaluation_id=run_id) + ev_run.end_run("FINISHED") + return _get_ai_studio_url(trace_destination=trace_destination, evaluation_id=ev_run.name) def _get_ai_studio_url(trace_destination: str, evaluation_id: str) -> str: @@ -232,7 +187,7 @@ def _apply_column_mapping(source_df: pd.DataFrame, mapping_config: dict, inplace if match is not None: pattern = match.group(1) if pattern.startswith(pattern_prefix): - map_from_key = pattern[len(pattern_prefix) :] + map_from_key = pattern[len(pattern_prefix):] elif pattern.startswith(run_outputs_prefix): # Target-generated columns always starts from .outputs. map_from_key = f"{Prefixes._TGT_OUTPUTS}{pattern[len(run_outputs_prefix) :]}" diff --git a/src/promptflow-evals/pyproject.toml b/src/promptflow-evals/pyproject.toml index 30a40a0c882..8d73dc026aa 100644 --- a/src/promptflow-evals/pyproject.toml +++ b/src/promptflow-evals/pyproject.toml @@ -44,9 +44,8 @@ promptflow-core = "<2.0.0" aiohttp_retry = ">=2.8.3" websocket-client = ">=1.2.0" jsonpath_ng = ">=1.5.0" +urllib3 = ">1.26.17" promptflow-azure = "<2.0.0" # Needed for remote tracking -mlflow = "<3.0.0" # Needed for remote tracking to log metrics -azureml-mlflow = "<2.0.0" # Needed for remote tracking to log metrics numpy = ">=1.22" [tool.poetry.group.dev.dependencies] diff --git a/src/promptflow-evals/tests/evals/e2etests/data/questions_answers.jsonl b/src/promptflow-evals/tests/evals/e2etests/data/questions_answers.jsonl new file mode 100644 index 00000000000..38fb6e7bc72 --- /dev/null +++ b/src/promptflow-evals/tests/evals/e2etests/data/questions_answers.jsonl @@ -0,0 +1 @@ +{"question":"How long is flight from Earth to LV-426?","answer":"There is nothing good there.", "ground_truth": "39 light years"} diff --git a/src/promptflow-evals/tests/evals/e2etests/test_evaluate.py b/src/promptflow-evals/tests/evals/e2etests/test_evaluate.py index 22faa2f073f..1840ce1ee8f 100644 --- a/src/promptflow-evals/tests/evals/e2etests/test_evaluate.py +++ b/src/promptflow-evals/tests/evals/e2etests/test_evaluate.py @@ -145,18 +145,16 @@ def test_evaluate_with_content_safety_evaluator(self, project_scope, data_file, assert 0 <= metrics.get("content_safety.self_harm_defect_rate") <= 1 assert 0 <= metrics.get("content_safety.hate_unfairness_defect_rate") <= 1 - @pytest.mark.parametrize( - "use_thread_pool,function,column", - [ - (True, answer_evaluator, "length"), - (False, answer_evaluator, "length"), - (True, answer_evaluator_int, "output"), - (False, answer_evaluator_int, "output"), - (True, answer_evaluator_int_dict, "42"), - (False, answer_evaluator_int_dict, "42"), - ], - ) - def test_evaluate_python_function(self, data_file, use_thread_pool, function, column): + @pytest.mark.parametrize('use_thread_pool,function,column', [ + (True, answer_evaluator, 'length'), + (False, answer_evaluator, 'length'), + (True, answer_evaluator_int, 'output'), + (False, answer_evaluator_int, 'output'), + (True, answer_evaluator_int_dict, "42"), + (False, answer_evaluator_int_dict, "42"), + ]) + def test_evaluate_python_function(self, data_file, use_thread_pool, + function, column): # data input_data = pd.read_json(data_file, lines=True) @@ -337,7 +335,7 @@ def test_evaluate_track_in_cloud( assert remote_run.properties["runType"] == "eval_run" assert remote_run.display_name == evaluation_name - @pytest.mark.skip(reason="az login in fixture is not working on ubuntu and mac.Works on windows") + @pytest.mark.skip(reason="az login in fixture is not working on ubuntu and mac. Works on windows") def test_evaluate_track_in_cloud_no_target( self, data_file, diff --git a/src/promptflow-evals/tests/evals/e2etests/test_metrics_upload.py b/src/promptflow-evals/tests/evals/e2etests/test_metrics_upload.py new file mode 100644 index 00000000000..0f5914583ba --- /dev/null +++ b/src/promptflow-evals/tests/evals/e2etests/test_metrics_upload.py @@ -0,0 +1,170 @@ +import json +import logging +import os +import pathlib +import pytest + +from unittest.mock import patch, MagicMock +from promptflow.evals.evaluate import _utils as ev_utils +from promptflow.evals.evaluate._eval_run import EvalRun +from promptflow.evals.evaluators._f1_score._f1_score import F1ScoreEvaluator +from promptflow.evals.evaluate._evaluate import evaluate +from promptflow.recording.record_mode import is_live + + +@pytest.fixture +def data_file(): + data_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data") + return os.path.join(data_path, "evaluate_test_data.jsonl") + + +@pytest.fixture +def questions_answers_file(): + data_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data") + return os.path.join(data_path, "questions_answers.jsonl") + + +@pytest.fixture +def questions_file(): + data_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data") + return os.path.join(data_path, "questions.jsonl") + + +@pytest.fixture +def setup_data(azure_pf_client, project_scope): + run = EvalRun( + run_name='test', + tracking_uri=( + 'https://eastus2.api.azureml.ms/mlflow/v2.0' + f'/subscriptions{project_scope["subscription_id"]}' + f'/resourceGroups/{project_scope["resource_group_name"]}' + '/providers/Microsoft.MachineLearningServices' + f'/workspaces/{project_scope["project_name"]}'), + subscription_id=project_scope["subscription_id"], + group_name=project_scope["resource_group_name"], + workspace_name=project_scope["project_name"], + ml_client=azure_pf_client._ml_client + ) + yield + run.end_run("FINISHED") + + +@pytest.mark.usefixtures("model_config", "recording_injection", "project_scope") +@pytest.mark.e2etest +class TestMetricsUpload(object): + """End to end tests to check how the metrics were uploaded to cloud.""" + + def _assert_no_errors_for_module(self, records, module_names): + """Check there are no errors in the log.""" + error_messages = [] + if records: + error_messages = [ + lg_rec.message for lg_rec in records if lg_rec.levelno == logging.ERROR and ( + lg_rec.name in module_names)] + assert not error_messages, '\n'.join(error_messages) + + @pytest.mark.usefixtures("vcr_recording") + def test_writing_to_run_history(self, setup_data, caplog): + """Test logging data to RunHistory service.""" + logger = logging.getLogger(ev_utils.__name__) + # All loggers, having promptflow. prefix will have "promptflow" logger + # as a parent. This logger does not propagate the logs and cannot be + # captured by caplog. Here we will skip this logger to capture logs. + logger.parent = logging.root + # Just for sanity check let us make sure that the logging actually works + mock_response = MagicMock() + mock_response.status_code = 418 + with patch('promptflow.evals.evaluate._eval_run.EvalRun.request_with_retry', return_value=mock_response): + ev_utils._write_properties_to_run_history({'test': 42}) + assert any(lg_rec.levelno == logging.ERROR for lg_rec in caplog.records), 'The error log was not captured!' + caplog.clear() + ev_utils._write_properties_to_run_history({'test': 42}) + self._assert_no_errors_for_module(caplog.records, [ev_utils.__name__]) + + @pytest.mark.usefixtures("vcr_recording") + def test_logging_metrics(self, setup_data, caplog): + """Test logging metrics.""" + logger = logging.getLogger(EvalRun.__module__) + # All loggers, having promptflow. prefix will have "promptflow" logger + # as a parent. This logger does not propagate the logs and cannot be + # captured by caplog. Here we will skip this logger to capture logs. + logger.parent = logging.root + ev_run = EvalRun.get_instance() + mock_response = MagicMock() + mock_response.status_code = 418 + with patch('promptflow.evals.evaluate._eval_run.EvalRun.request_with_retry', return_value=mock_response): + ev_run.log_metric('f1', 0.54) + assert any(lg_rec.levelno == logging.ERROR for lg_rec in caplog.records), 'The error log was not captured!' + caplog.clear() + ev_run.log_metric('f1', 0.54) + self._assert_no_errors_for_module(caplog.records, EvalRun.__module__) + + @pytest.mark.usefixtures("vcr_recording") + def test_log_artifact(self, setup_data, caplog, tmp_path): + """Test uploading artifact to the service.""" + logger = logging.getLogger(EvalRun.__module__) + # All loggers, having promptflow. prefix will have "promptflow" logger + # as a parent. This logger does not propagate the logs and cannot be + # captured by caplog. Here we will skip this logger to capture logs. + logger.parent = logging.root + ev_run = EvalRun.get_instance() + mock_response = MagicMock() + mock_response.status_code = 418 + with open(os.path.join(tmp_path, 'test.json'), 'w') as fp: + json.dump({'f1': 0.5}, fp) + os.makedirs(os.path.join(tmp_path, 'internal_dir'), exist_ok=True) + with open(os.path.join(tmp_path, 'internal_dir', 'test.json'), 'w') as fp: + json.dump({'internal_f1': 0.6}, fp) + with patch('promptflow.evals.evaluate._eval_run.EvalRun.request_with_retry', return_value=mock_response): + ev_run.log_artifact(tmp_path) + assert any(lg_rec.levelno == logging.ERROR for lg_rec in caplog.records), 'The error log was not captured!' + caplog.clear() + ev_run.log_artifact(tmp_path) + self._assert_no_errors_for_module(caplog.records, EvalRun.__module__) + + @pytest.mark.skipif(condition=not is_live(), + reason="promptflow run create files with random names, which cannot be recorded.") + @pytest.mark.usefixtures("vcr_recording") + def test_e2e_run_target_fn(self, caplog, project_scope, questions_answers_file): + """Test evaluation run logging.""" + # We cannot define target in this file as pytest will load + # all modules in test folder and target_fn will be imported from the first + # module named test_evaluate and it will be a different module in unit test + # folder. By keeping function in separate file we guarantee, it will be loaded + # from there. + from .target_fn import target_fn + + f1_score_eval = F1ScoreEvaluator() + # We need the deterministic name of a run, however it cannot be recorded + # into database more then once or the database may be non writable. + # By this reason we will cancel writing to database by mocking it. + # Please uncomment this line for the local tests + # with patch('promptflow._sdk.entities._run.Run._dump'): + evaluate( + data=questions_answers_file, + target=target_fn, + evaluators={"f1": f1_score_eval}, + azure_ai_project=project_scope, + _run_name='eval_test_run2' + ) + self._assert_no_errors_for_module(caplog.records, (ev_utils.__name__, EvalRun.__module__)) + + @pytest.mark.usefixtures("vcr_recording") + def test_e2e_run(self, caplog, project_scope, questions_answers_file): + """Test evaluation run logging.""" + # Make sure that the URL ending in TraceSessions is in the recording, it is not always being recorded. + # To record this test please modify the YAML file. It is missing "mlFlowTrackingUri" property by default. + # Search URIs: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups + # /00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + # In the BLOB SAS URI change sktid to 00000000-0000-0000-0000-000000000000 + # Add the tracking URI to properties dictionary with the key "mlFlowTrackingUri": + # azureml://eastus2.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/ + # resourceGroups/00000000-0000-0000-0000-000000000000/providers/Microsoft.MachineLearningServices/ + # workspaces/00000 + f1_score_eval = F1ScoreEvaluator() + evaluate( + data=questions_answers_file, + evaluators={"f1": f1_score_eval}, + azure_ai_project=project_scope + ) + self._assert_no_errors_for_module(caplog.records, (ev_utils.__name__, EvalRun.__module__)) diff --git a/src/promptflow-evals/tests/evals/unittests/test_eval_run.py b/src/promptflow-evals/tests/evals/unittests/test_eval_run.py new file mode 100644 index 00000000000..0288b1ddce4 --- /dev/null +++ b/src/promptflow-evals/tests/evals/unittests/test_eval_run.py @@ -0,0 +1,458 @@ +import json +import logging +import os +import pytest + +from unittest.mock import MagicMock, patch + +from promptflow.evals.evaluate._eval_run import Singleton, EvalRun +from uuid import uuid4 +import promptflow.evals.evaluate._utils as ev_utils + + +@pytest.fixture +def setup_data(): + """Make sure, we will destroy the EvalRun instance as it is singleton.""" + yield + Singleton._instances.clear() + + +@pytest.mark.unittest +class TestEvalRun: + """Unit tests for the eval-run object.""" + + @pytest.mark.parametrize( + 'status,should_raise', + [ + ("KILLED", False), + ("WRONG_STATUS", True), + ("FINISHED", False), + ("FAILED", False) + ] + ) + def test_end_raises(self, setup_data, status, should_raise, caplog): + """Test that end run raises exception if incorrect status is set.""" + mock_session = MagicMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + 'run': { + "info": { + "run_id": str(uuid4()), + "experiment_id": str(uuid4()), + } + } + } + mock_session.request.return_value = mock_response + with patch('promptflow.evals.evaluate._eval_run.requests.Session', return_value=mock_session): + run = EvalRun( + run_name=None, + tracking_uri='www.microsoft.com', + subscription_id='mock', + group_name='mock', + workspace_name='mock', + ml_client=MagicMock() + ) + if should_raise: + with pytest.raises(ValueError) as cm: + run.end_run(status) + assert status in cm.value.args[0] + else: + run.end_run(status) + assert len(caplog.records) == 0 + + def test_run_logs_if_terminated(self, setup_data, caplog): + """Test that run warn user if we are trying to terminate it twice.""" + mock_session = MagicMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + 'run': { + "info": { + "run_id": str(uuid4()), + "experiment_id": str(uuid4()), + } + } + } + mock_session.request.return_value = mock_response + with patch('promptflow.evals.evaluate._eval_run.requests.Session', return_value=mock_session): + logger = logging.getLogger(EvalRun.__module__) + # All loggers, having promptflow. prefix will have "promptflow" logger + # as a parent. This logger does not propagate the logs and cannot be + # captured by caplog. Here we will skip this logger to capture logs. + logger.parent = logging.root + run = EvalRun( + run_name=None, + tracking_uri='www.microsoft.com', + subscription_id='mock', + group_name='mock', + workspace_name='mock', + ml_client=MagicMock() + ) + run.end_run("KILLED") + run.end_run("KILLED") + assert len(caplog.records) == 1 + assert "Unable to stop run because it was already terminated." in caplog.records[0].message + + def test_end_logs_if_fails(self, setup_data, caplog): + """Test that if the terminal status setting was failed, it is logged.""" + mock_session = MagicMock() + mock_response_start = MagicMock() + mock_response_start.status_code = 200 + mock_response_start.json.return_value = { + 'run': { + "info": { + "run_id": str(uuid4()), + "experiment_id": str(uuid4()), + } + } + } + mock_response_end = MagicMock() + mock_response_end.status_code = 500 + mock_session.request.side_effect = [mock_response_start, mock_response_end] + with patch('promptflow.evals.evaluate._eval_run.requests.Session', return_value=mock_session): + logger = logging.getLogger(EvalRun.__module__) + # All loggers, having promptflow. prefix will have "promptflow" logger + # as a parent. This logger does not propagate the logs and cannot be + # captured by caplog. Here we will skip this logger to capture logs. + logger.parent = logging.root + run = EvalRun( + run_name=None, + tracking_uri='www.microsoft.com', + subscription_id='mock', + group_name='mock', + workspace_name='mock', + ml_client=MagicMock() + ) + run.end_run("FINISHED") + assert len(caplog.records) == 1 + assert "Unable to terminate the run." in caplog.records[0].message + + def test_start_run_fails(self, setup_data, caplog): + """Test that there are log messges if run was not started.""" + mock_session = MagicMock() + mock_response_start = MagicMock() + mock_response_start.status_code = 500 + mock_response_start.text = "Mock internal service error." + mock_session.request.return_value = mock_response_start + with patch('promptflow.evals.evaluate._eval_run.requests.Session', return_value=mock_session): + logger = logging.getLogger(EvalRun.__module__) + # All loggers, having promptflow. prefix will have "promptflow" logger + # as a parent. This logger does not propagate the logs and cannot be + # captured by caplog. Here we will skip this logger to capture logs. + logger.parent = logging.root + run = EvalRun( + run_name=None, + tracking_uri='www.microsoft.com', + subscription_id='mock', + group_name='mock', + workspace_name='mock', + ml_client=MagicMock() + ) + assert len(caplog.records) == 1 + assert "500" in caplog.records[0].message + assert mock_response_start.text in caplog.records[0].message + assert 'The results will be saved locally' in caplog.records[0].message + caplog.clear() + # Log artifact + run.log_artifact('test') + assert len(caplog.records) == 1 + assert "Unable to log artifact because the run failed to start." in caplog.records[0].message + caplog.clear() + # Log metric + run.log_metric('a', 42) + assert len(caplog.records) == 1 + assert "Unable to log metric because the run failed to start." in caplog.records[0].message + caplog.clear() + # End run + run.end_run("FINISHED") + assert len(caplog.records) == 1 + assert "Unable to stop run because the run failed to start." in caplog.records[0].message + caplog.clear() + + @pytest.mark.parametrize( + 'destroy_run,runs_are_the_same', + [ + (False, True), + (True, False) + ] + ) + @patch('promptflow.evals.evaluate._eval_run.requests.Session') + def test_singleton(self, mock_session_cls, setup_data, destroy_run, runs_are_the_same): + """Test that the EvalRun is actually a singleton.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.side_effect = [ + { + 'run': { + "info": { + "run_id": str(uuid4()), + "experiment_id": str(uuid4()), + } + } + }, + { + 'run': { + "info": { + "run_id": str(uuid4()), + "experiment_id": str(uuid4()), + } + } + }, + ] + mock_session = MagicMock() + mock_session.request.return_value = mock_response + mock_session_cls.return_value = mock_session + run = EvalRun( + run_name='run', + tracking_uri='www.microsoft.com', + subscription_id='mock', + group_name='mock', + workspace_name='mock', + ml_client=MagicMock() + ) + id1 = id(run) + if destroy_run: + run.end_run("FINISHED") + id2 = id( + EvalRun( + run_name='run', + tracking_uri='www.microsoft.com', + subscription_id='mock', + group_name='mock', + workspace_name='mock', + ml_client=MagicMock() + ) + ) + assert (id1 == id2) == runs_are_the_same + + @patch('promptflow.evals.evaluate._eval_run.requests.Session') + def test_run_name(self, mock_session_cls, setup_data): + """Test that the run name is the same as ID if name is not given.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + 'run': { + "info": { + "run_id": str(uuid4()), + "experiment_id": str(uuid4()), + } + } + } + mock_session = MagicMock() + mock_session.request.return_value = mock_response + mock_session_cls.return_value = mock_session + run = EvalRun( + run_name=None, + tracking_uri='www.microsoft.com', + subscription_id='mock', + group_name='mock', + workspace_name='mock', + ml_client=MagicMock() + ) + assert run.info.run_id == mock_response.json.return_value['run']['info']['run_id'] + assert run.info.experiment_id == mock_response.json.return_value[ + 'run']['info']['experiment_id'] + assert run.name == run.info.run_id + + @patch('promptflow.evals.evaluate._eval_run.requests.Session') + def test_run_with_name(self, mock_session_cls, setup_data): + """Test that the run name is not the same as id if it is given.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + 'run': { + "info": { + "run_id": str(uuid4()), + "experiment_id": str(uuid4()), + } + } + } + mock_session = MagicMock() + mock_session.request.return_value = mock_response + mock_session_cls.return_value = mock_session + run = EvalRun( + run_name='test', + tracking_uri='www.microsoft.com', + subscription_id='mock', + group_name='mock', + workspace_name='mock', + ml_client=MagicMock() + ) + assert run.info.run_id == mock_response.json.return_value['run']['info']['run_id'] + assert run.info.experiment_id == mock_response.json.return_value[ + 'run']['info']['experiment_id'] + assert run.name == 'test' + assert run.name != run.info.run_id + + @patch('promptflow.evals.evaluate._eval_run.requests.Session') + def test_get_urls(self, mock_session_cls, setup_data): + """Test getting url-s from eval run.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + 'run': { + "info": { + "run_id": str(uuid4()), + "experiment_id": str(uuid4()), + } + } + } + mock_session = MagicMock() + mock_session.request.return_value = mock_response + mock_session_cls.return_value = mock_session + run = EvalRun( + run_name='test', + tracking_uri=( + 'https://region.api.azureml.ms/mlflow/v2.0/subscriptions' + '/000000-0000-0000-0000-0000000/resourceGroups/mock-rg-region' + '/providers/Microsoft.MachineLearningServices' + '/workspaces/mock-ws-region'), + subscription_id='000000-0000-0000-0000-0000000', + group_name='mock-rg-region', + workspace_name='mock-ws-region', + ml_client=MagicMock() + ) + assert run.get_run_history_uri() == ( + 'https://region.api.azureml.ms/history/v1.0/subscriptions' + '/000000-0000-0000-0000-0000000/resourceGroups/mock-rg-region' + '/providers/Microsoft.MachineLearningServices' + '/workspaces/mock-ws-region/experimentids/' + f'{run.info.experiment_id}/runs/{run.info.run_id}'), 'Wrong RunHistory URL' + assert run.get_artifacts_uri() == ( + 'https://region.api.azureml.ms/history/v1.0/subscriptions' + '/000000-0000-0000-0000-0000000/resourceGroups/mock-rg-region' + '/providers/Microsoft.MachineLearningServices' + '/workspaces/mock-ws-region/experimentids/' + f'{run.info.experiment_id}/runs/{run.info.run_id}' + '/artifacts/batch/metadata' + ), 'Wrong Artifacts URL' + assert run.get_metrics_url() == ( + 'https://region.api.azureml.ms/mlflow/v2.0/subscriptions' + '/000000-0000-0000-0000-0000000/resourceGroups/mock-rg-region' + '/providers/Microsoft.MachineLearningServices' + '/workspaces/mock-ws-region/api/2.0/mlflow/runs/log-metric' + ), 'Wrong Metrics URL' + + @pytest.mark.parametrize( + 'log_function,expected_str', + [ + ('log_artifact', 'allocate Blob for the artifact'), + ('log_metric', 'save metrics') + ] + ) + def test_log_artifacts_logs_error( + self, + setup_data, tmp_path, caplog, + log_function, expected_str + ): + """Test that the error is logged.""" + mock_session = MagicMock() + mock_create_response = MagicMock() + mock_create_response.status_code = 200 + mock_create_response.json.return_value = { + 'run': { + "info": { + "run_id": str(uuid4()), + "experiment_id": str(uuid4()), + } + } + } + mock_response = MagicMock() + mock_response.status_code = 404 + mock_response.text = 'Mock not found error.' + + if log_function == 'log_artifact': + with open(os.path.join(tmp_path, 'test.json'), 'w') as fp: + json.dump({'f1': 0.5}, fp) + mock_session.request.side_effect = [ + mock_create_response, + mock_response + ] + with patch('promptflow.evals.evaluate._eval_run.requests.Session', return_value=mock_session): + run = EvalRun( + run_name='test', + tracking_uri=( + 'https://region.api.azureml.ms/mlflow/v2.0/subscriptions' + '/000000-0000-0000-0000-0000000/resourceGroups/mock-rg-region' + '/providers/Microsoft.MachineLearningServices' + '/workspaces/mock-ws-region'), + subscription_id='000000-0000-0000-0000-0000000', + group_name='mock-rg-region', + workspace_name='mock-ws-region', + ml_client=MagicMock() + ) + + logger = logging.getLogger(EvalRun.__module__) + # All loggers, having promptflow. prefix will have "promptflow" logger + # as a parent. This logger does not propagate the logs and cannot be + # captured by caplog. Here we will skip this logger to capture logs. + logger.parent = logging.root + fn = getattr(run, log_function) + if log_function == 'log_artifact': + kwargs = {'artifact_folder': tmp_path} + else: + kwargs = {'key': 'f1', 'value': 0.5} + fn(**kwargs) + assert len(caplog.records) == 1 + assert mock_response.text in caplog.records[0].message + assert '404' in caplog.records[0].message + assert expected_str in caplog.records[0].message + + @pytest.mark.parametrize( + 'dir_exists,expected_error', [ + (True, "The path to the artifact is empty."), + (False, "The path to the artifact is either not a directory or does not exist.") + ] + ) + def test_wrong_artifact_path(self, tmp_path, caplog, dir_exists, expected_error): + """Test that if artifact path is empty, or dies not exist we are logging the error.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + 'run': { + "info": { + "run_id": str(uuid4()), + "experiment_id": str(uuid4()), + } + } + } + mock_session = MagicMock() + mock_session.request.return_value = mock_response + with patch('promptflow.evals.evaluate._eval_run.requests.Session', return_value=mock_session): + run = EvalRun( + run_name='test', + tracking_uri=( + 'https://region.api.azureml.ms/mlflow/v2.0/subscriptions' + '/000000-0000-0000-0000-0000000/resourceGroups/mock-rg-region' + '/providers/Microsoft.MachineLearningServices' + '/workspaces/mock-ws-region'), + subscription_id='000000-0000-0000-0000-0000000', + group_name='mock-rg-region', + workspace_name='mock-ws-region', + ml_client=MagicMock() + ) + logger = logging.getLogger(EvalRun.__module__) + # All loggers, having promptflow. prefix will have "promptflow" logger + # as a parent. This logger does not propagate the logs and cannot be + # captured by caplog. Here we will skip this logger to capture logs. + logger.parent = logging.root + artifact_folder = tmp_path if dir_exists else "wrong_path_567" + run.log_artifact(artifact_folder) + assert len(caplog.records) == 1 + assert expected_error in caplog.records[0].message + + def test_log_metrics_and_instance_results_logs_error(self, caplog): + """Test that we are logging the error when there is no trace destination.""" + logger = logging.getLogger(ev_utils.__name__) + # All loggers, having promptflow. prefix will have "promptflow" logger + # as a parent. This logger does not propagate the logs and cannot be + # captured by caplog. Here we will skip this logger to capture logs. + logger.parent = logging.root + ev_utils._log_metrics_and_instance_results( + metrics=None, + instance_results=None, + trace_destination=None, + run=None) + assert len(caplog.records) == 1 + assert "Unable to log traces as trace destination was not defined." in caplog.records[0].message diff --git a/src/promptflow-evals/tests/evals/unittests/test_get_trace_destination_config.py b/src/promptflow-evals/tests/evals/unittests/test_get_trace_destination_config.py deleted file mode 100644 index 66fc209e3d8..00000000000 --- a/src/promptflow-evals/tests/evals/unittests/test_get_trace_destination_config.py +++ /dev/null @@ -1,32 +0,0 @@ -import pytest -from unittest.mock import patch -from promptflow.evals.evaluate._utils import _get_trace_destination_config - - -@pytest.fixture -def patch_config_validation(): - with patch("promptflow._sdk._configuration.Configuration._validate", return_value=None): - yield - - -@pytest.mark.unittest -class TestGetTraceDestinationConfig: - @pytest.mark.parametrize("trace_destination, expected_trace_destination", - [ - ("None", None), - ("none", None), - ("NONE", None), - ("NoNe", None), - ]) - def test_get_trace_destination_config(self, trace_destination, expected_trace_destination): - with patch("promptflow._sdk._configuration.Configuration.get_trace_destination", - return_value=trace_destination): - assert _get_trace_destination_config(None) == expected_trace_destination - - def test_get_trace_destination_config_with_override(self, patch_config_validation): - trace_destination = ("azureml://subscriptions/subscription-id/resourceGroups/resource-group-name/providers" - "/Microsoft.MachineLearningServices/workspaces/test_get_trace_destination_config") - global_trace_destination_value = _get_trace_destination_config(None) - overidden_trace_destination_value = _get_trace_destination_config(trace_destination) - assert global_trace_destination_value != overidden_trace_destination_value - assert overidden_trace_destination_value == trace_destination diff --git a/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_e2e_run.yaml b/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_e2e_run.yaml new file mode 100644 index 00000000000..68e4435c612 --- /dev/null +++ b/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_e2e_run.yaml @@ -0,0 +1,596 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.11.5 + (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus2", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus2.api.azureml.ms/discovery", "mlFlowTrackingUri": "azureml://eastus2.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000000-0000-0000-0000-000000000000/providers/Microsoft.MachineLearningServices/workspaces/00000"}}' + headers: + cache-control: + - no-cache + content-length: + - '2978' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.031' + status: + code: 200 + message: OK +- request: + body: '[{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "2024-06-06T23:20:59.838896Z", + "sampleRate": 100.0, "iKey": "00000000-0000-0000-0000-000000000000", "tags": + {"foo": "bar"}}]' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '927' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azuremonitorclient/unknown Python/3.11.5 (Windows-10-10.0.22631-SP0) + method: POST + uri: https://dc.services.visualstudio.com/v2.1/track + response: + body: + string: '{"itemsReceived": 1, "itemsAccepted": 0, "appId": null, "errors": [{"index": + 0, "statusCode": 307, "message": "Ingestion is allowed only from stamp specific + endpoint - Location: https://eastus-8.in.applicationinsights.azure.com/v2.1/track"}]}' + headers: + cache-control: + - max-age=604800 + content-type: + - application/json; charset=utf-8 + location: + - https://eastus-8.in.applicationinsights.azure.com/v2.1/track + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + status: + code: 307 + message: Temporary Redirect +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.11.5 + (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus2", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus2.api.azureml.ms/discovery", "mlFlowTrackingUri": "azureml://eastus2.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000000-0000-0000-0000-000000000000/providers/Microsoft.MachineLearningServices/workspaces/00000"}}' + headers: + cache-control: + - no-cache + content-length: + - '2978' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.023' + status: + code: 200 + message: OK +- request: + body: '[{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "2024-06-06T23:20:59.838896Z", + "sampleRate": 100.0, "iKey": "00000000-0000-0000-0000-000000000000", "tags": + {"foo": "bar"}}]' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '927' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azuremonitorclient/unknown Python/3.11.5 (Windows-10-10.0.22631-SP0) + method: POST + uri: https://eastus-8.in.applicationinsights.azure.com/v2.1/track + response: + body: + string: '{"itemsReceived": 1, "itemsAccepted": 1, "appId": null, "errors": []}' + headers: + content-type: + - application/json; charset=utf-8 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/0.0.1.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://eastus2.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/TraceSessions + response: + body: + string: '{"entityId": "a1391af3-9f38-48e8-862f-db3846e81dd4", "traceCosmosConfiguration": + "None", "traceCosmosStatus": "Initialized", "accountEndpoint": "", "databaseName": + "PromptFlowTraceSession", "resourceArmId": "", "resourceType": 1}' + headers: + connection: + - keep-alive + content-length: + - '486' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.533' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.11.5 + (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus2", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus2.api.azureml.ms/discovery", "mlFlowTrackingUri": "azureml://eastus2.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000000-0000-0000-0000-000000000000/providers/Microsoft.MachineLearningServices/workspaces/00000"}}' + headers: + cache-control: + - no-cache + content-length: + - '2978' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.027' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.11.5 + (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus2", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus2.api.azureml.ms/discovery", "mlFlowTrackingUri": "azureml://eastus2.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000000-0000-0000-0000-000000000000/providers/Microsoft.MachineLearningServices/workspaces/00000"}}' + headers: + cache-control: + - no-cache + content-length: + - '2978' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.031' + status: + code: 200 + message: OK +- request: + body: '{"experiment_id": "0", "user_id": "promptflow-evals", "start_time": "1717563256142", + "tags": [{"key": "mlflow.user", "value": "promptflow-evals"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '145' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api/2.0/mlflow/runs/create + response: + body: + string: '{"run": {"info": {"run_uuid": "b797dd76-c228-4a0c-8ba4-0d41f4dce7ad", + "experiment_id": "5ea666e7-ae55-4060-a8a3-71e6e03fab9c", "run_name": "nice_floor_hb1b1h9t", + "user_id": "00000000-0000-0000-0000-000000000000", "status": "RUNNING", "start_time": + "1717733443454", "artifact_uri": "azureml://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/experiments/0/runs/b797dd76-c228-4a0c-8ba4-0d41f4dce7ad/artifacts", + "lifecycle_stage": "active", "run_id": "b797dd76-c228-4a0c-8ba4-0d41f4dce7ad"}, + "data": {"tags": [{"key": "mlflow.user", "value": "promptflow-evals"}, {"key": + "mlflow.rootRunId", "value": "b797dd76-c228-4a0c-8ba4-0d41f4dce7ad"}, {"key": + "mlflow.runName", "value": "nice_floor_hb1b1h9t"}, {"key": "mlflow.user", + "value": "Nikolay Rovinskiy"}]}, "inputs": {}}}' + headers: + connection: + - keep-alive + content-length: + - '931' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.351' + status: + code: 200 + message: OK +- request: + body: '{"paths": [{"path": "evaluation_results/eval_results.jsonl"}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '62' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/history/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/experimentids/5ea666e7-ae55-4060-a8a3-71e6e03fab9c/runs/b797dd76-c228-4a0c-8ba4-0d41f4dce7ad/artifacts/batch/metadata + response: + body: + string: '{"artifacts": {"evaluation_results/eval_results.jsonl": {"artifactId": + "ExperimentRun/dcid.b797dd76-c228-4a0c-8ba4-0d41f4dce7ad/evaluation_results/eval_results.jsonl", + "origin": "ExperimentRun", "container": "dcid.b797dd76-c228-4a0c-8ba4-0d41f4dce7ad", + "path": "evaluation_results/eval_results.jsonl", "etag": null, "createdTime": + "2024-06-07T04:10:50.0015468+00:00", "dataPath": null, "tags": {}}}, "artifactContentInformation": + {"evaluation_results/eval_results.jsonl": {"contentUri": "https://nirovinswseast5028951403.blob.core.windows.net/azureml/ExperimentRun/dcid.b797dd76-c228-4a0c-8ba4-0d41f4dce7ad/evaluation_results/eval_results.jsonl?sv=2019-07-07&sr=b&sig=dRHHxYUsJlve7j73P4xhC2zP4lGoae7e0O%2FBSwjm8vE%3D&skoid=f4777969-da99-42e7-a95c-5f6396e15e83&sktid=00000000-0000-0000-0000-000000000000&skt=2024-06-07T04%3A00%3A49Z&ske=2024-06-08T12%3A10%3A49Z&sks=b&skv=2019-07-07&st=2024-06-07T04%3A00%3A49Z&se=2024-06-08T04%3A10%3A49Z&sp=rcw", + "origin": "ExperimentRun", "container": "dcid.b797dd76-c228-4a0c-8ba4-0d41f4dce7ad", + "path": "evaluation_results/eval_results.jsonl", "tags": {}}}, "errors": {}}' + headers: + connection: + - keep-alive + content-length: + - '1222' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '1.682' + status: + code: 200 + message: OK +- request: + body: "{\"inputs.question\":\"How long is flight from Earth to LV-426?\",\"inputs.answer\":\"There + is nothing good there.\",\"inputs.ground_truth\":\"39 light years\",\"outputs.f1.f1_score\":0.0,\"line_number\":0}\r\n" + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '192' + Content-Type: + - application/octet-stream + If-None-Match: + - '*' + User-Agent: + - azsdk-python-storage-blob/12.20.0 Python/3.11.5 (Windows-10-10.0.22631-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Fri, 07 Jun 2024 04:10:50 GMT + x-ms-version: + - '2024-05-04' + method: PUT + uri: https://nirovinswseast5028951403.blob.core.windows.net/azureml/ExperimentRun/dcid.b797dd76-c228-4a0c-8ba4-0d41f4dce7ad/evaluation_results/eval_results.jsonl?se=2024-06-08T04%3A10%3A49Z&sig=dRHHxYUsJlve7j73P4xhC2zP4lGoae7e0O%2FBSwjm8vE%3D&ske=2024-06-08T12%3A10%3A49Z&skoid=f4777969-da99-42e7-a95c-5f6396e15e83&sks=b&skt=2024-06-07T04%3A00%3A49Z&sktid=00000000-0000-0000-0000-000000000000&skv=2019-07-07&sp=rcw&sr=b&st=2024-06-07T04%3A00%3A49Z&sv=2019-07-07 + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - G3n2KMFFaqFbUGDeX34xXw== + last-modified: + - Fri, 07 Jun 2024 04:10:51 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - 0Myc02lzbG0= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2024-05-04' + status: + code: 201 + message: Created +- request: + body: '{"runId": "evals_e2etests_target_fn_wqo0_peh_20240606_102622_386974", "properties": + {"_azureml.evaluation_run": "azure-ai-generative-parent", "_azureml.evaluate_artifacts": + "[{\"path\": \"eval_results.jsonl\", \"type\": \"table\"}]", "isEvaluatorRun": + "true"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '240' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: PATCH + uri: https://eastus2.api.azureml.ms/history/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/experimentids/5ea666e7-ae55-4060-a8a3-71e6e03fab9c/runs/b797dd76-c228-4a0c-8ba4-0d41f4dce7ad + response: + body: + string: '{"runNumber": 1717733445, "rootRunId": "b797dd76-c228-4a0c-8ba4-0d41f4dce7ad", + "createdUtc": "2024-06-07T04:10:43.454+00:00", "createdBy": {"userObjectId": + "00000000-0000-0000-0000-000000000000", "userPuId": "1003BFFDA8A4D0E7", "userIdp": + null, "userAltSecId": null, "userIss": "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", + "userTenantId": "00000000-0000-0000-0000-000000000000", "userName": "Nikolay + Rovinskiy", "upn": "username@microsoft.com"}, "userId": "00000000-0000-0000-0000-000000000000", + "token": "eyJhbGciOiJSUzI1NiIsImtpZCI6Ijk3QTcyRUQyOUNFMjMwMTQwQjVGNzFEOTkyODk2NzBDRDRGNEJFMzUiLCJ0eXAiOiJKV1QifQ.eyJyb2xlIjoiQ29udHJpYnV0b3IiLCJzY29wZSI6Ii9zdWJzY3JpcHRpb25zLzZhNmZmZjAwLTQ0NjQtNGVhYi1hNmIxLTBiNTMzYzcyMDJlMC9yZXNvdXJjZUdyb3Vwcy9uaXJvdmlucy1yZy1lYXN0dXMvcHJvdmlkZXJzL01pY3Jvc29mdC5NYWNoaW5lTGVhcm5pbmdTZXJ2aWNlcy93b3Jrc3BhY2VzL25pcm92aW5zLXdzLWVhc3R1cyIsImFjY291bnRpZCI6IjAwMDAwMDAwLTAwMDAtMDAwMC0wMDAwLTAwMDAwMDAwMDAwMCIsIndvcmtzcGFjZUlkIjoiYTEzOTFhZjMtOWYzOC00OGU4LTg2MmYtZGIzODQ2ZTgxZGQ0IiwicHJvamVjdGlkIjoiMDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAwIiwiZGlzY292ZXJ5IjoidXJpOi8vZGlzY292ZXJ5dXJpLyIsInRpZCI6IjcyZjk4OGJmLTg2ZjEtNDFhZi05MWFiLTJkN2NkMDExZGI0NyIsIm9pZCI6Ijc4NzViYWE2LTAyOGQtNDljOC1iZDRkLWE3NzJhYTRkYzAxMyIsInB1aWQiOiIxMDAzQkZGREE4QTREMEU3IiwiaXNzIjoiYXp1cmVtbCIsImFwcGlkIjoiTmlrb2xheSBSb3ZpbnNraXkiLCJleHAiOjE3MTk1NTUwNDUsImF1ZCI6ImF6dXJlbWwifQ.kaxuNrkEz4ARPy6HrRPEacX_xSZtHq4yBavn8TzVcP5UgaOCPpMPDWP20kddt2heommexMarDJ22MNzCB_S4x6ZbAfbg8TqBc7eohTq67Dirl2G5hfU0LEbgGxBPsaHSgdwPcVpnV6BGb1EfFctKJk9AegfY9hBNnZjDWJxTwdDxkNnM0PrwZAHqNqEyDPCVe2BGeogIN2n9IVPf80uMkcpTJQr_SyV1KV6uhUS-8975Xj9-wK9-1_XVBOJQcWX3-kkRRl-dfGFk59w7eFJ7eku6h9mm0G7Ft2qzMiR4Zmu3yG8T5cCQYDX8k2OaxIhHqJi1awjux_BvYJuwoXfg5A", + "tokenExpiryTimeUtc": "2024-06-28T06:10:45.8846867+00:00", "error": null, + "warnings": null, "revision": 2, "statusRevision": 0, "runUuid": "9f45dee8-d84a-49b7-8b2c-e93f685c6237", + "parentRunUuid": null, "rootRunUuid": "9f45dee8-d84a-49b7-8b2c-e93f685c6237", + "lastStartTimeUtc": "2024-06-07T04:10:43.454+00:00", "currentComputeTime": + "00:00:00", "computeDuration": null, "effectiveStartTimeUtc": "2024-06-07T04:10:43.454+00:00", + "lastModifiedBy": {"userObjectId": "00000000-0000-0000-0000-000000000000", + "userPuId": "1003BFFDA8A4D0E7", "userIdp": null, "userAltSecId": null, "userIss": + "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", "userTenantId": + "00000000-0000-0000-0000-000000000000", "userName": "Nikolay Rovinskiy", "upn": + "username@microsoft.com"}, "lastModifiedUtc": "2024-06-07T04:10:53.3381884+00:00", + "duration": null, "cancelationReason": null, "currentAttemptId": 1, "runId": + "b797dd76-c228-4a0c-8ba4-0d41f4dce7ad", "parentRunId": null, "experimentId": + "00000000-0000-0000-0000-000000000000", "status": "Running", "startTimeUtc": + "2024-06-07T04:10:43.454+00:00", "endTimeUtc": null, "scheduleId": null, "displayName": + "nice_floor_hb1b1h9t", "name": null, "dataContainerId": "dcid.b797dd76-c228-4a0c-8ba4-0d41f4dce7ad", + "description": null, "hidden": false, "runType": null, "runTypeV2": {"orchestrator": + null, "traits": ["mlflow"], "attribution": null, "computeType": null}, "properties": + {"mlflow.artifactUri": "azureml://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/experiments/0/runs/b797dd76-c228-4a0c-8ba4-0d41f4dce7ad/artifacts", + "_azureml.evaluation_run": "azure-ai-generative-parent", "_azureml.evaluate_artifacts": + "[{\"path\": \"eval_results.jsonl\", \"type\": \"table\"}]", "isEvaluatorRun": + "true"}, "parameters": {}, "actionUris": {}, "scriptName": null, "target": + null, "uniqueChildRunComputeTargets": [], "tags": {"mlflow.user": "promptflow-evals"}, + "settings": {}, "services": {}, "inputDatasets": [], "outputDatasets": [], + "runDefinition": null, "jobSpecification": null, "primaryMetricName": null, + "createdFrom": null, "cancelUri": null, "completeUri": null, "diagnosticsUri": + null, "computeRequest": null, "compute": null, "retainForLifetimeOfWorkspace": + false, "queueingInfo": null, "inputs": null, "outputs": null}' + headers: + connection: + - keep-alive + content-length: + - '4410' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.039' + status: + code: 200 + message: OK +- request: + body: '{"run_uuid": "b797dd76-c228-4a0c-8ba4-0d41f4dce7ad", "key": "f1.f1_score", + "value": 0.0, "timestamp": "1717563256242", "step": 0, "run_id": "b797dd76-c228-4a0c-8ba4-0d41f4dce7ad"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '177' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api/2.0/mlflow/runs/log-metric + response: + body: + string: '{}' + headers: + connection: + - keep-alive + content-length: + - '3' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-request-time: + - '0.717' + status: + code: 200 + message: OK +- request: + body: '{"run_uuid": "b797dd76-c228-4a0c-8ba4-0d41f4dce7ad", "status": "FINISHED", + "end_time": "1717563261483", "run_id": "b797dd76-c228-4a0c-8ba4-0d41f4dce7ad"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '151' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api/2.0/mlflow/runs/update + response: + body: + string: '{"run_info": {"run_uuid": "b797dd76-c228-4a0c-8ba4-0d41f4dce7ad", "experiment_id": + "5ea666e7-ae55-4060-a8a3-71e6e03fab9c", "run_name": "nice_floor_hb1b1h9t", + "user_id": "00000000-0000-0000-0000-000000000000", "status": "FINISHED", "start_time": + "1717733443454", "end_time": "1717733458668", "lifecycle_stage": "active", + "run_id": "b797dd76-c228-4a0c-8ba4-0d41f4dce7ad"}}' + headers: + connection: + - keep-alive + content-length: + - '374' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.551' + status: + code: 200 + message: OK +version: 1 diff --git a/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_e2e_run_target_fn.yaml b/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_e2e_run_target_fn.yaml new file mode 100644 index 00000000000..b763c94c1f0 --- /dev/null +++ b/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_e2e_run_target_fn.yaml @@ -0,0 +1,1749 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.11.5 + (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus2", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus2.api.azureml.ms/discovery", "mlFlowTrackingUri": "azureml://eastus2.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000000-0000-0000-0000-000000000000/providers/Microsoft.MachineLearningServices/workspaces/00000"}}' + headers: + cache-control: + - no-cache + content-length: + - '2978' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.023' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.11.5 + (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus2", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus2.api.azureml.ms/discovery", "mlFlowTrackingUri": "azureml://eastus2.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000000-0000-0000-0000-000000000000/providers/Microsoft.MachineLearningServices/workspaces/00000"}}' + headers: + cache-control: + - no-cache + content-length: + - '2978' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.028' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/0.0.1.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://eastus2.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/TraceSessions + response: + body: + string: '{"entityId": "a1391af3-9f38-48e8-862f-db3846e81dd4", "traceCosmosConfiguration": + "None", "traceCosmosStatus": "Initialized", "accountEndpoint": "", "databaseName": + "PromptFlowTraceSession", "resourceArmId": "", "resourceType": 1}' + headers: + connection: + - keep-alive + content-length: + - '486' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.732' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.11.5 + (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus2", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus2.api.azureml.ms/discovery", "mlFlowTrackingUri": "azureml://eastus2.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000000-0000-0000-0000-000000000000/providers/Microsoft.MachineLearningServices/workspaces/00000"}}' + headers: + cache-control: + - no-cache + content-length: + - '2978' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.023' + status: + code: 200 + message: OK +- request: + body: '[{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "2024-06-06T23:20:59.838896Z", + "sampleRate": 100.0, "iKey": "00000000-0000-0000-0000-000000000000", "tags": + {"foo": "bar"}}]' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '1019' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azuremonitorclient/unknown Python/3.11.5 (Windows-10-10.0.22631-SP0) + method: POST + uri: https://eastus-8.in.applicationinsights.azure.com/v2.1/track + response: + body: + string: '{"itemsReceived": 1, "itemsAccepted": 1, "appId": null, "errors": []}' + headers: + content-type: + - application/json; charset=utf-8 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.11.5 + (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores?count=30&isDefault=true&orderByAsc=false + response: + body: + string: '{"value": [{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, + "systemData": {"createdAt": "2024-01-19T20:15:50.0791508+00:00", "createdBy": + "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": + "2024-01-19T20:15:50.6629868+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application"}}]}' + headers: + cache-control: + - no-cache + content-length: + - '1390' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.046' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.11.5 + (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores?count=30&isDefault=true&orderByAsc=false + response: + body: + string: '{"value": [{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + true, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity"}, + "systemData": {"createdAt": "2024-01-19T20:15:50.0791508+00:00", "createdBy": + "779301c0-18b2-4cdc-801b-a0a3368fee0a", "createdByType": "Application", "lastModifiedAt": + "2024-01-19T20:15:50.6629868+00:00", "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application"}}]}' + headers: + cache-control: + - no-cache + content-length: + - '1390' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.046' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.11.5 + (Windows-10-10.0.22631-SP0) + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore/listSecrets + response: + body: + string: '{"secretsType": "AccountKey", "key": "dGhpcyBpcyBmYWtlIGtleQ=="}' + headers: + cache-control: + - no-cache + content-length: + - '134' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.154' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.11.5 + (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceartifactstore + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceartifactstore", + "name": "workspaceartifactstore", "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": {"description": null, "tags": null, "properties": null, "isDefault": + false, "credentials": {"credentialsType": "AccountKey"}, "intellectualProperty": + null, "subscriptionId": "00000000-0000-0000-0000-000000000000", "resourceGroup": + "00000", "datastoreType": "AzureBlob", "accountName": "fake_account_name", + "containerName": "fake-container-name", "endpoint": "core.windows.net", "protocol": + "https", "serviceDataAccessAuthIdentity": "None"}, "systemData": {"createdAt": + "2024-01-19T20:15:50.0844646+00:00", "createdBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "createdByType": "Application", "lastModifiedAt": "2024-01-19T20:15:50.7065161+00:00", + "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", "lastModifiedByType": + "Application"}}' + headers: + cache-control: + - no-cache + content-length: + - '1130' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.058' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.11.5 + (Windows-10-10.0.22631-SP0) + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceartifactstore/listSecrets + response: + body: + string: '{"secretsType": "AccountKey", "key": "dGhpcyBpcyBmYWtlIGtleQ=="}' + headers: + cache-control: + - no-cache + content-length: + - '134' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.081' + status: + code: 200 + message: OK +- request: + body: '[{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "2024-06-06T23:20:59.838896Z", + "sampleRate": 100.0, "iKey": "00000000-0000-0000-0000-000000000000", "tags": + {"foo": "bar"}}]' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '6917' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azuremonitorclient/unknown Python/3.11.5 (Windows-10-10.0.22631-SP0) + method: POST + uri: https://eastus-8.in.applicationinsights.azure.com/v2.1/track + response: + body: + string: '{"itemsReceived": 7, "itemsAccepted": 7, "appId": null, "errors": []}' + headers: + content-type: + - application/json; charset=utf-8 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + status: + code: 200 + message: OK +- request: + body: '{"batch_size": 25}' + headers: + Accept: + - application/xml + Content-Length: + - '18' + Content-Type: + - application/octet-stream + User-Agent: + - azsdk-python-storage-blob/12.20.0 Python/3.11.5 (Windows-10-10.0.22631-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Fri, 07 Jun 2024 05:04:49 GMT + x-ms-version: + - '2024-05-04' + method: PUT + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/promptflow/PromptFlowArtifacts/eval_test_run2/meta.json + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - /u1NXUpgXMFDmZEw835qnw== + last-modified: + - Fri, 07 Jun 2024 05:04:50 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - /9Grig3aMP8= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2024-05-04' + status: + code: 201 + message: Created +- request: + body: "{\"line_number\": 0, \"run_info\": {\"run_id\": \"eval_test_run2_0\", \"status\": + \"Completed\", \"error\": null, \"inputs\": {\"question\": \"How long is flight + from Earth to LV-426?\", \"line_number\": 0}, \"output\": {\"answer\": \"There + is nothing good there.\"}, \"metrics\": null, \"request\": null, \"parent_run_id\": + \"eval_test_run2\", \"root_run_id\": \"eval_test_run2\", \"source_run_id\": + null, \"flow_id\": \"default_flow_id\", \"start_time\": \"2024-06-07T05:04:43.934422Z\", + \"end_time\": \"2024-06-07T05:04:43.935422Z\", \"index\": 0, \"api_calls\": + [{\"name\": \"target_fn\", \"type\": \"Flow\", \"inputs\": {\"question\": \"How + long is flight from Earth to LV-426?\"}, \"output\": {\"answer\": \"There is + nothing good there.\"}, \"start_time\": 1717761883.934422, \"end_time\": 1717761883.935422, + \"error\": null, \"children\": [], \"node_name\": null, \"parent_id\": \"\", + \"id\": \"d9f9165b-40a9-4526-9327-9d127aac969d\", \"function\": \"target_fn\", + \"system_metrics\": {}}], \"name\": \"\", \"description\": \"\", \"tags\": null, + \"system_metrics\": {\"duration\": 0.001}, \"result\": null, \"upload_metrics\": + false, \"otel_trace_id\": \"0x00000000000000000000000000000000\", \"message_format\": + \"basic\"}, \"start_time\": \"2024-06-07T05:04:43.934422\", \"end_time\": \"2024-06-07T05:04:43.935422\", + \"name\": \"\", \"description\": \"\", \"status\": \"Completed\", \"tags\": + null}\r\n" + headers: + Accept: + - application/xml + Content-Length: + - '1258' + Content-Type: + - application/octet-stream + User-Agent: + - azsdk-python-storage-blob/12.20.0 Python/3.11.5 (Windows-10-10.0.22631-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Fri, 07 Jun 2024 05:04:49 GMT + x-ms-version: + - '2024-05-04' + method: PUT + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/promptflow/PromptFlowArtifacts/eval_test_run2/flow_artifacts/000000000_000000024.jsonl + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - iwYi6P5gIj5Z1UM8jWas+g== + last-modified: + - Fri, 07 Jun 2024 05:04:50 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - 8oUeojLzuiU= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2024-05-04' + status: + code: 201 + message: Created +- request: + body: "{\"line_number\": 0, \"answer\": \"There is nothing good there.\"}\r\n" + headers: + Accept: + - application/xml + Content-Length: + - '62' + Content-Type: + - application/octet-stream + User-Agent: + - azsdk-python-storage-blob/12.20.0 Python/3.11.5 (Windows-10-10.0.22631-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Fri, 07 Jun 2024 05:04:49 GMT + x-ms-version: + - '2024-05-04' + method: PUT + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/promptflow/PromptFlowArtifacts/eval_test_run2/flow_outputs/output.jsonl + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - hMBCgK7/vClMM6xTJHnV2g== + last-modified: + - Fri, 07 Jun 2024 05:04:50 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - YJNOWI6/rXE= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2024-05-04' + status: + code: 201 + message: Created +- request: + body: "2024-06-06 22:04:30 -0700 26420 execution.bulk INFO Current thread + is not main thread, skip signal handler registration in BatchEngine.\r\n2024-06-06 + 22:04:30 -0700 26420 execution WARNING Starting run without column + mapping may lead to unexpected results. Please consult the following documentation + for more information: https://aka.ms/pf/column-mapping\r\n2024-06-06 22:04:30 + -0700 26420 execution.bulk INFO Current system's available memory + is 3280.8515625MB, memory consumption of current process is 362.94140625MB, + estimated available worker count is 3280.8515625/362.94140625 = 9\r\n2024-06-06 + 22:04:30 -0700 26420 execution.bulk INFO Set process count to 1 by + taking the minimum value among the factors of {'default_worker_count': 4, 'row_count': + 1, 'estimated_worker_count_based_on_memory_usage': 9}.\r\n2024-06-06 22:04:37 + -0700 26420 execution.bulk INFO Process name(MockSpawnProcess-4)-Process + id(13844)-Line number(0) start execution.\r\n2024-06-06 22:04:43 -0700 26420 + execution.bulk INFO Process name(MockSpawnProcess-4)-Process id(13844)-Line + number(0) completed.\r\n2024-06-06 22:04:44 -0700 26420 execution.bulk INFO + \ Finished 1 / 1 lines.\r\n2024-06-06 22:04:44 -0700 26420 execution.bulk + \ INFO Average execution time for completed lines: 7.05 seconds. Estimated + time for incomplete lines: 0.0 seconds.\r\n2024-06-06 22:04:44 -0700 26420 + execution.bulk INFO The thread monitoring the process [13844-MockSpawnProcess-4] + will be terminated.\r\n2024-06-06 22:04:44 -0700 13844 execution.bulk INFO + \ The process [13844] has received a terminate signal.\r\n2024-06-06 22:04:44 + -0700 26420 execution.bulk INFO Process 13844 terminated.\r\n" + headers: + Accept: + - application/xml + Content-Length: + - '1737' + Content-Type: + - application/octet-stream + User-Agent: + - azsdk-python-storage-blob/12.20.0 Python/3.11.5 (Windows-10-10.0.22631-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Fri, 07 Jun 2024 05:04:49 GMT + x-ms-version: + - '2024-05-04' + method: PUT + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/ExperimentRun/dcid.eval_test_run2/logs/azureml/executionlogs.txt + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - IOahykFUz8RuuO9QomnD6g== + last-modified: + - Fri, 07 Jun 2024 05:04:50 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - 1g+obuUj1cQ= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2024-05-04' + status: + code: 201 + message: Created +- request: + body: "{\"line_number\": 0, \"status\": \"Completed\", \"inputs.question\": \"How + long is flight from Earth to LV-426?\", \"inputs.line_number\": 0, \"answer\": + \"There is nothing good there.\"}\r\n" + headers: + Accept: + - application/xml + Content-Length: + - '173' + Content-Type: + - application/octet-stream + User-Agent: + - azsdk-python-storage-blob/12.20.0 Python/3.11.5 (Windows-10-10.0.22631-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Fri, 07 Jun 2024 05:04:49 GMT + x-ms-version: + - '2024-05-04' + method: PUT + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/promptflow/PromptFlowArtifacts/eval_test_run2/instance_results.jsonl + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - 4vS4Od1H9cNd+Y1uoTFvVA== + last-modified: + - Fri, 07 Jun 2024 05:04:50 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - VmrFaAfcg6I= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2024-05-04' + status: + code: 201 + message: Created +- request: + body: "entry: evals.e2etests.target_fn:target_fn\r\ninputs:\r\n question:\r\n + \ type: string\r\noutputs:\r\n output:\r\n type: string\r\n" + headers: + Accept: + - application/xml + Content-Length: + - '122' + Content-Type: + - application/octet-stream + User-Agent: + - azsdk-python-storage-blob/12.20.0 Python/3.11.5 (Windows-10-10.0.22631-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Fri, 07 Jun 2024 05:04:49 GMT + x-ms-version: + - '2024-05-04' + method: PUT + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/runs/eval_test_run2/flow.flex.yaml + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - 3i/HHriLVrIcnduTB06OfQ== + last-modified: + - Fri, 07 Jun 2024 05:04:50 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - 4GD4bAgr6qM= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2024-05-04' + status: + code: 201 + message: Created +- request: + body: null + headers: + Accept: + - application/xml + Content-Length: + - '0' + Content-Type: + - application/octet-stream + User-Agent: + - azsdk-python-storage-blob/12.20.0 Python/3.11.5 (Windows-10-10.0.22631-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Fri, 07 Jun 2024 05:04:49 GMT + x-ms-version: + - '2024-05-04' + method: PUT + uri: https://fake_account_name.blob.core.windows.net/fake-container-name/promptflow/PromptFlowArtifacts/eval_test_run2/flow_logs/000000000.log + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - 1B2M2Y8AsgTpgAmY7PhCfg== + last-modified: + - Fri, 07 Jun 2024 05:04:50 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - AAAAAAAAAAA= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2024-05-04' + status: + code: 201 + message: Created +- request: + body: '[{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "2024-06-06T23:20:59.838896Z", + "sampleRate": 100.0, "iKey": "00000000-0000-0000-0000-000000000000", "tags": + {"foo": "bar"}}]' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '1709' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azuremonitorclient/unknown Python/3.11.5 (Windows-10-10.0.22631-SP0) + method: POST + uri: https://eastus-8.in.applicationinsights.azure.com/v2.1/track + response: + body: + string: '{"itemsReceived": 1, "itemsAccepted": 1, "appId": null, "errors": []}' + headers: + content-type: + - application/json; charset=utf-8 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + status: + code: 200 + message: OK +- request: + body: '{"origin": "ExperimentRun", "container": "dcid.eval_test_run2", "path": + "instance_results.jsonl", "dataPath": {"dataStoreName": "workspaceblobstore", + "relativePath": "promptflow/PromptFlowArtifacts/eval_test_run2/instance_results.jsonl"}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '238' + content-type: + - application/json + host: + - eastus2.api.azureml.ms + user-agent: + - promptflow/1.12.0.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/artifact/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/artifacts/register + response: + body: + string: '{"artifactId": "ExperimentRun/dcid.eval_test_run2/instance_results.jsonl", + "origin": "ExperimentRun", "container": "dcid.eval_test_run2", "path": "instance_results.jsonl", + "etag": "\"5d009b12-0000-0200-0000-666294f20000\"", "createdTime": "2024-06-07T05:04:50.8904269+00:00", + "dataPath": {"dataStoreName": "workspaceblobstore", "relativePath": "promptflow/PromptFlowArtifacts/eval_test_run2/instance_results.jsonl"}, + "tags": {}}' + headers: + connection: + - keep-alive + content-length: + - '458' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.168' + status: + code: 200 + message: OK +- request: + body: '{"runId": "evals_e2etests_target_fn_wqo0_peh_20240606_102622_386974", "runDisplayName": + "eval_test_run2", "properties": {"_azureml.evaluate_artifacts": "[{\"path\": + \"instance_results.jsonl\", \"type\": \"table\"}]", "azureml.promptflow.total_tokens": + "0", "azureml.promptflow.completion_tokens": "0", "azureml.promptflow.prompt_tokens": + "0", "runType": "eval_run"}, "runExperimentName": "", "runDisplayNameGenerationType": + "UserProvidedMacro", "outputDataStore": "workspaceblobstore", "flowArtifactsRootPath": + "promptflow/PromptFlowArtifacts/eval_test_run2", "logFileRelativePath": "ExperimentRun/dcid.eval_test_run2/logs/azureml/executionlogs.txt", + "flowDefinitionDataStoreName": "workspaceblobstore", "flowDefinitionBlobPath": + "runs/eval_test_run2/flow.flex.yaml", "runStatus": "Completed", "endTimeUtc": + "2024-06-06T22:04:44.898856Z", "startTimeUtc": "2024-06-06T22:04:12.535374Z"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '886' + Content-Type: + - application/json + User-Agent: + - promptflow-azure-sdk/0.0.1.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22631-SP0) + method: POST + uri: https://eastus2.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/create + response: + body: + string: '"eval_test_run2"' + headers: + connection: + - keep-alive + content-length: + - '16' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-request-time: + - '1.681' + status: + code: 200 + message: OK +- request: + body: '{"RunId": "eval_test_run2", "OutputName": "debug_info", "Type": "UriFolder", + "Uri": "azureml://subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/workspaces/00000/datastores/workspaceblobstore/paths/promptflow/PromptFlowArtifacts/eval_test_run2"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '293' + content-type: + - application/json + host: + - eastus2.api.azureml.ms + user-agent: + - promptflow/1.12.0.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/data/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/dataversion/createUnregisteredOutput + response: + body: + string: '{"dataContainer": {"assetId": "azureml://locations/eastus2/workspaces/00000/data/azureml_eval_test_run2_output_data_debug_info", + "name": "azureml_eval_test_run2_output_data_debug_info", "dataType": "UriFolder", + "mutableProps": {"description": null, "tags": null, "isArchived": false}, + "isRegistered": false}, "entityMetadata": {"etag": "\"4e07d68b-0000-0200-0000-666294fc0000\"", + "createdTime": "2024-06-07T05:05:00.3113071+00:00", "modifiedTime": "2024-06-07T05:05:00.3490177+00:00", + "createdBy": {"userObjectId": "00000000-0000-0000-0000-000000000000", "userPuId": + "1003BFFDA8A4D0E7", "userIdp": null, "userAltSecId": null, "userIss": "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", + "userTenantId": "00000000-0000-0000-0000-000000000000", "userName": "Nikolay + Rovinskiy", "upn": "username@microsoft.com"}, "modifiedBy": null}, "latestVersion": + {"dataVersion": {"assetId": "azureml://locations/eastus2/workspaces/00000/data/azureml_eval_test_run2_output_data_debug_info/versions/1", + "dataContainerName": "azureml_eval_test_run2_output_data_debug_info", "dataType": + "UriFolder", "dataUri": "azureml://subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/workspaces/00000/datastores/workspaceblobstore/paths/promptflow/PromptFlowArtifacts/eval_test_run2/", + "versionId": "1", "mutableProps": {"dataExpiryTime": null, "description": + null, "tags": null, "isArchived": false, "stage": "Logged", "autoDeleteSetting": + null}, "referencedDataUris": null, "properties": null, "initialAssetId": "azureml://locations/eastus2/workspaces/00000/data/azureml_eval_test_run2_output_data_debug_info/versions/1", + "isRegistered": false, "runId": "eval_test_run2", "originAssetId": null}, + "entityMetadata": {"etag": "\"4e07d38b-0000-0200-0000-666294fc0000\"", "createdTime": + "2024-06-07T05:05:00.3307157+00:00", "modifiedTime": "2024-06-07T05:05:00.337042+00:00", + "createdBy": {"userObjectId": "00000000-0000-0000-0000-000000000000", "userPuId": + "1003BFFDA8A4D0E7", "userIdp": null, "userAltSecId": null, "userIss": "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", + "userTenantId": "00000000-0000-0000-0000-000000000000", "userName": "Nikolay + Rovinskiy", "upn": "username@microsoft.com"}, "modifiedBy": null}, "legacyDatasetId": + "5361e13b-2bdf-4096-90e4-bf629985a32f", "isV2": true, "legacyDatasetType": + null, "legacyDataflowType": null, "legacyDataflow": null, "legacySavedDatasetId": + null, "putAssetLROResponseDto": null}, "isV2": true, "nextVersionId": "2", + "legacyDatasetId": "5361e13b-2bdf-4096-90e4-bf629985a32f", "legacyDatasetType": + null, "putAssetLROResponseDto": null}' + headers: + connection: + - keep-alive + content-length: + - '3153' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.283' + status: + code: 200 + message: OK +- request: + body: '{"RunId": "eval_test_run2", "OutputName": "flow_outputs", "Type": "UriFolder", + "Uri": "azureml://subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/workspaces/00000/datastores/workspaceblobstore/paths/promptflow/PromptFlowArtifacts/eval_test_run2/flow_outputs"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '308' + content-type: + - application/json + host: + - eastus2.api.azureml.ms + user-agent: + - promptflow/1.12.0.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/data/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/dataversion/createUnregisteredOutput + response: + body: + string: '{"dataContainer": {"assetId": "azureml://locations/eastus2/workspaces/00000/data/azureml_eval_test_run2_output_data_flow_outputs", + "name": "azureml_eval_test_run2_output_data_flow_outputs", "dataType": "UriFolder", + "mutableProps": {"description": null, "tags": null, "isArchived": false}, + "isRegistered": false}, "entityMetadata": {"etag": "\"4e07d98b-0000-0200-0000-666294fc0000\"", + "createdTime": "2024-06-07T05:05:00.3431307+00:00", "modifiedTime": "2024-06-07T05:05:00.3803677+00:00", + "createdBy": {"userObjectId": "00000000-0000-0000-0000-000000000000", "userPuId": + "1003BFFDA8A4D0E7", "userIdp": null, "userAltSecId": null, "userIss": "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", + "userTenantId": "00000000-0000-0000-0000-000000000000", "userName": "Nikolay + Rovinskiy", "upn": "username@microsoft.com"}, "modifiedBy": null}, "latestVersion": + {"dataVersion": {"assetId": "azureml://locations/eastus2/workspaces/00000/data/azureml_eval_test_run2_output_data_flow_outputs/versions/1", + "dataContainerName": "azureml_eval_test_run2_output_data_flow_outputs", "dataType": + "UriFolder", "dataUri": "azureml://subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/workspaces/00000/datastores/workspaceblobstore/paths/promptflow/PromptFlowArtifacts/eval_test_run2/flow_outputs/", + "versionId": "1", "mutableProps": {"dataExpiryTime": null, "description": + null, "tags": null, "isArchived": false, "stage": "Logged", "autoDeleteSetting": + null}, "referencedDataUris": null, "properties": null, "initialAssetId": "azureml://locations/eastus2/workspaces/00000/data/azureml_eval_test_run2_output_data_flow_outputs/versions/1", + "isRegistered": false, "runId": "eval_test_run2", "originAssetId": null}, + "entityMetadata": {"etag": "\"4e07d78b-0000-0200-0000-666294fc0000\"", "createdTime": + "2024-06-07T05:05:00.3585305+00:00", "modifiedTime": "2024-06-07T05:05:00.3679494+00:00", + "createdBy": {"userObjectId": "00000000-0000-0000-0000-000000000000", "userPuId": + "1003BFFDA8A4D0E7", "userIdp": null, "userAltSecId": null, "userIss": "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", + "userTenantId": "00000000-0000-0000-0000-000000000000", "userName": "Nikolay + Rovinskiy", "upn": "username@microsoft.com"}, "modifiedBy": null}, "legacyDatasetId": + "7546c328-6c89-4625-baa5-a20c92d2fc55", "isV2": true, "legacyDatasetType": + null, "legacyDataflowType": null, "legacyDataflow": null, "legacySavedDatasetId": + null, "putAssetLROResponseDto": null}, "isV2": true, "nextVersionId": "2", + "legacyDatasetId": "7546c328-6c89-4625-baa5-a20c92d2fc55", "legacyDatasetType": + null, "putAssetLROResponseDto": null}' + headers: + connection: + - keep-alive + content-length: + - '3177' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.300' + status: + code: 200 + message: OK +- request: + body: '{"values": [{"name": "__pf__.lines.failed", "columns": {"__pf__.lines.failed": + "Double"}, "properties": {"uxMetricType": "azureml.v1.scalar"}, "value": [{"data": + {"__pf__.lines.failed": 0.0}, "step": 0}]}]}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '206' + content-type: + - application/json + host: + - eastus2.api.azureml.ms + user-agent: + - promptflow/1.12.0.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/metric/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/runs/eval_test_run2/batchsync + response: + body: + string: '{"errors": [], "batchTrackingId": "15572068"}' + headers: + connection: + - keep-alive + content-length: + - '51' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-request-time: + - '0.628' + status: + code: 200 + message: OK +- request: + body: '{"values": [{"name": "__pf__.lines.completed", "columns": {"__pf__.lines.completed": + "Double"}, "properties": {"uxMetricType": "azureml.v1.scalar"}, "value": [{"data": + {"__pf__.lines.completed": 1.0}, "step": 0}]}]}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '215' + content-type: + - application/json + host: + - eastus2.api.azureml.ms + user-agent: + - promptflow/1.12.0.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/metric/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/runs/eval_test_run2/batchsync + response: + body: + string: '{"errors": [], "batchTrackingId": "15572067"}' + headers: + connection: + - keep-alive + content-length: + - '51' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-request-time: + - '0.457' + status: + code: 200 + message: OK +- request: + body: '{"Outputs": {"debug_info": {"assetId": "azureml://locations/eastus2/workspaces/00000/data/azureml_eval_test_run2_output_data_debug_info/versions/1", + "type": "UriFolder"}, "flow_outputs": {"assetId": "azureml://locations/eastus2/workspaces/00000/data/azureml_eval_test_run2_output_data_flow_outputs/versions/1", + "type": "UriFolder"}}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '395' + content-type: + - application/json + host: + - eastus2.api.azureml.ms + user-agent: + - promptflow/1.12.0.dev0 + method: PATCH + uri: https://eastus2.api.azureml.ms/history/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/runs/eval_test_run2 + response: + body: + string: '{"runNumber": 1717736694, "rootRunId": "eval_test_run2", "createdUtc": + "2024-06-06T22:04:12.535374+00:00", "createdBy": {"userObjectId": "00000000-0000-0000-0000-000000000000", + "userPuId": "1003BFFDA8A4D0E7", "userIdp": null, "userAltSecId": null, "userIss": + "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", "userTenantId": + "00000000-0000-0000-0000-000000000000", "userName": "Nikolay Rovinskiy", "upn": + null}, "userId": "00000000-0000-0000-0000-000000000000", "token": null, "tokenExpiryTimeUtc": + null, "error": null, "warnings": null, "revision": 1, "statusRevision": 0, + "runUuid": "4c193ea8-996b-41b4-9744-bbedeecc4b79", "parentRunUuid": null, + "rootRunUuid": "4c193ea8-996b-41b4-9744-bbedeecc4b79", "lastStartTimeUtc": + "2024-06-06T22:04:12.535374+00:00", "currentComputeTime": "00:00:00", "computeDuration": + "00:00:00", "effectiveStartTimeUtc": "2024-06-06T22:04:12.535374+00:00", "lastModifiedBy": + {"userObjectId": "00000000-0000-0000-0000-000000000000", "userPuId": "1003BFFDA8A4D0E7", + "userIdp": null, "userAltSecId": null, "userIss": "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", + "userTenantId": "00000000-0000-0000-0000-000000000000", "userName": "Nikolay + Rovinskiy", "upn": "username@microsoft.com"}, "lastModifiedUtc": "2024-06-07T05:05:02.1321249+00:00", + "duration": "00:00:32.3634820", "cancelationReason": null, "currentAttemptId": + 1, "runId": "eval_test_run2", "parentRunId": null, "experimentId": "00000000-0000-0000-0000-000000000000", + "status": "Completed", "startTimeUtc": "2024-06-06T22:04:12.535374+00:00", + "endTimeUtc": "2024-06-06T22:04:44.898856+00:00", "scheduleId": null, "displayName": + "eval_test_run2", "name": null, "dataContainerId": "dcid.eval_test_run2", + "description": null, "hidden": false, "runType": "azureml.promptflow.FlowRun", + "runTypeV2": {"orchestrator": null, "traits": [], "attribution": "PromptFlow", + "computeType": null}, "properties": {"_azureml.evaluate_artifacts": "[{\"path\": + \"instance_results.jsonl\", \"type\": \"table\"}]", "azureml.promptflow.total_tokens": + "0", "azureml.promptflow.completion_tokens": "0", "azureml.promptflow.prompt_tokens": + "0", "runType": "eval_run", "azureml.promptflow.runtime_name": null, "azureml.promptflow.disable_trace": + "false", "azureml.promptflow.local_to_cloud": "true", "azureml.promptflow.snapshot_id": + "f733a4d3-edf9-4372-95df-ef5a5aba4033", "_azureml.evaluation_run": "promptflow.BatchRun"}, + "parameters": {}, "actionUris": {}, "scriptName": null, "target": null, "uniqueChildRunComputeTargets": + [], "tags": {}, "settings": {}, "services": {}, "inputDatasets": [], "outputDatasets": + [], "runDefinition": null, "jobSpecification": null, "primaryMetricName": + null, "createdFrom": null, "cancelUri": null, "completeUri": null, "diagnosticsUri": + null, "computeRequest": null, "compute": null, "retainForLifetimeOfWorkspace": + false, "queueingInfo": null, "inputs": null, "outputs": null}' + headers: + connection: + - keep-alive + content-length: + - '3167' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.037' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/0.0.1.dev0 azsdk-python-azuremachinelearningdesignerserviceclient/unknown + Python/3.11.5 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://eastus2.api.azureml.ms/flow/api/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/BulkRuns/eval_test_run2 + response: + body: + string: '{"flowGraph": {"inputs": {"question": {"type": "string", "is_chat_input": + false}}, "outputs": {"output": {"type": "string", "evaluation_only": false, + "is_chat_output": false}}}, "flowRunResourceId": "azureml://locations/eastus2/workspaces/00000/flows/eval_test_run2/flowRuns/eval_test_run2", + "flowRunId": "eval_test_run2", "flowRunDisplayName": "eval_test_run2", "flowRunType": + "FlowRun", "flowType": "Default", "outputDatastoreName": "workspaceblobstore", + "childRunBasePath": "promptflow/PromptFlowArtifacts/eval_test_run2/flow_artifacts", + "flowDagFileRelativePath": "flow.flex.yaml", "flowSnapshotId": "f733a4d3-edf9-4372-95df-ef5a5aba4033", + "studioPortalEndpoint": "https://ml.azure.com/runs/eval_test_run2?wsid=/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "studioPortalTraceEndpoint": "https://ml.azure.com/prompts/trace/run/eval_test_run2/details?wsid=/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/00000/providers/Microsoft.MachineLearningServices/workspaces/nirovins-ws-eastus"}' + headers: + connection: + - keep-alive + content-length: + - '1130' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.208' + status: + code: 200 + message: OK +- request: + body: '[{"ver": 1, "name": "Microsoft.ApplicationInsights.Event", "time": "2024-06-06T23:20:59.838896Z", + "sampleRate": 100.0, "iKey": "00000000-0000-0000-0000-000000000000", "tags": + {"foo": "bar"}}]' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '17185' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azuremonitorclient/unknown Python/3.11.5 (Windows-10-10.0.22631-SP0) + method: POST + uri: https://eastus-8.in.applicationinsights.azure.com/v2.1/track + response: + body: + string: '{"itemsReceived": 17, "itemsAccepted": 17, "appId": null, "errors": + []}' + headers: + content-type: + - application/json; charset=utf-8 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.11.5 + (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus2", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus2.api.azureml.ms/discovery", "mlFlowTrackingUri": "azureml://eastus2.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000000-0000-0000-0000-000000000000/providers/Microsoft.MachineLearningServices/workspaces/00000"}}' + headers: + cache-control: + - no-cache + content-length: + - '2978' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.023' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.11.5 + (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus2", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus2.api.azureml.ms/discovery", "mlFlowTrackingUri": "azureml://eastus2.api.azureml.ms/mlflow/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000000-0000-0000-0000-000000000000/providers/Microsoft.MachineLearningServices/workspaces/00000"}}' + headers: + cache-control: + - no-cache + content-length: + - '2978' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.027' + status: + code: 200 + message: OK +- request: + body: '{"experiment_id": "0", "user_id": "promptflow-evals", "start_time": "1717563256142", + "tags": [{"key": "mlflow.user", "value": "promptflow-evals"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '145' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api/2.0/mlflow/runs/create + response: + body: + string: '{"run": {"info": {"run_uuid": "feb068c0-fea0-4f0f-b5d0-b2a8960bd97b", + "experiment_id": "5ea666e7-ae55-4060-a8a3-71e6e03fab9c", "run_name": "green_island_wpnhqk6x", + "user_id": "00000000-0000-0000-0000-000000000000", "status": "RUNNING", "start_time": + "1717736706998", "artifact_uri": "azureml://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/experiments/0/runs/feb068c0-fea0-4f0f-b5d0-b2a8960bd97b/artifacts", + "lifecycle_stage": "active", "run_id": "feb068c0-fea0-4f0f-b5d0-b2a8960bd97b"}, + "data": {"tags": [{"key": "mlflow.user", "value": "promptflow-evals"}, {"key": + "mlflow.rootRunId", "value": "feb068c0-fea0-4f0f-b5d0-b2a8960bd97b"}, {"key": + "mlflow.runName", "value": "green_island_wpnhqk6x"}, {"key": "mlflow.user", + "value": "Nikolay Rovinskiy"}]}, "inputs": {}}}' + headers: + connection: + - keep-alive + content-length: + - '935' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.283' + status: + code: 200 + message: OK +- request: + body: '{"paths": [{"path": "evaluation_results/eval_results.jsonl"}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '62' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/history/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/experimentids/5ea666e7-ae55-4060-a8a3-71e6e03fab9c/runs/feb068c0-fea0-4f0f-b5d0-b2a8960bd97b/artifacts/batch/metadata + response: + body: + string: '{"artifacts": {"evaluation_results/eval_results.jsonl": {"artifactId": + "ExperimentRun/dcid.feb068c0-fea0-4f0f-b5d0-b2a8960bd97b/evaluation_results/eval_results.jsonl", + "origin": "ExperimentRun", "container": "dcid.feb068c0-fea0-4f0f-b5d0-b2a8960bd97b", + "path": "evaluation_results/eval_results.jsonl", "etag": null, "createdTime": + "2024-06-07T05:05:12.0813112+00:00", "dataPath": null, "tags": {}}}, "artifactContentInformation": + {"evaluation_results/eval_results.jsonl": {"contentUri": "https://nirovinswseast5028951403.blob.core.windows.net/azureml/ExperimentRun/dcid.feb068c0-fea0-4f0f-b5d0-b2a8960bd97b/evaluation_results/eval_results.jsonl?sv=2019-07-07&sr=b&sig=T1w%2FvOwQ7OeBofj382CauzoPuOLbzNILBEPmMhwX%2Bzs%3D&skoid=f4777969-da99-42e7-a95c-5f6396e15e83&sktid=00000000-0000-0000-0000-000000000000&skt=2024-06-07T04%3A00%3A49Z&ske=2024-06-08T12%3A10%3A49Z&sks=b&skv=2019-07-07&st=2024-06-07T04%3A55%3A12Z&se=2024-06-08T05%3A05%3A12Z&sp=rcw", + "origin": "ExperimentRun", "container": "dcid.feb068c0-fea0-4f0f-b5d0-b2a8960bd97b", + "path": "evaluation_results/eval_results.jsonl", "tags": {}}}, "errors": {}}' + headers: + connection: + - keep-alive + content-length: + - '1224' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.500' + status: + code: 200 + message: OK +- request: + body: "{\"outputs.answer\":\"There is nothing good there.\",\"inputs.question\":\"How + long is flight from Earth to LV-426?\",\"inputs.answer\":\"There is nothing + good there.\",\"inputs.ground_truth\":\"39 light years\",\"outputs.f1.f1_score\":0.0,\"line_number\":0}\r\n" + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '240' + Content-Type: + - application/octet-stream + If-None-Match: + - '*' + User-Agent: + - azsdk-python-storage-blob/12.20.0 Python/3.11.5 (Windows-10-10.0.22631-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Fri, 07 Jun 2024 05:05:12 GMT + x-ms-version: + - '2024-05-04' + method: PUT + uri: https://nirovinswseast5028951403.blob.core.windows.net/azureml/ExperimentRun/dcid.feb068c0-fea0-4f0f-b5d0-b2a8960bd97b/evaluation_results/eval_results.jsonl?sv=2019-07-07&sr=b&sig=T1w%2FvOwQ7OeBofj382CauzoPuOLbzNILBEPmMhwX%2Bzs%3D&skoid=f4777969-da99-42e7-a95c-5f6396e15e83&sktid=00000000-0000-0000-0000-000000000000&skt=2024-06-07T04%3A00%3A49Z&ske=2024-06-08T12%3A10%3A49Z&sks=b&skv=2019-07-07&st=2024-06-07T04%3A55%3A12Z&se=2024-06-08T05%3A05%3A12Z&sp=rcw + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - 1UmSuZ3GCYngVifGaO/Ipw== + last-modified: + - Fri, 07 Jun 2024 05:05:13 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - nhBnI+JvENs= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2024-05-04' + status: + code: 201 + message: Created +- request: + body: '{"runId": "evals_e2etests_target_fn_wqo0_peh_20240606_102622_386974", "properties": + {"_azureml.evaluation_run": "azure-ai-generative-parent", "_azureml.evaluate_artifacts": + "[{\"path\": \"eval_results.jsonl\", \"type\": \"table\"}]", "isEvaluatorRun": + "true"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '240' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: PATCH + uri: https://eastus2.api.azureml.ms/history/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/experimentids/5ea666e7-ae55-4060-a8a3-71e6e03fab9c/runs/feb068c0-fea0-4f0f-b5d0-b2a8960bd97b + response: + body: + string: '{"runNumber": 1717736709, "rootRunId": "feb068c0-fea0-4f0f-b5d0-b2a8960bd97b", + "createdUtc": "2024-06-07T05:05:06.998+00:00", "createdBy": {"userObjectId": + "00000000-0000-0000-0000-000000000000", "userPuId": "1003BFFDA8A4D0E7", "userIdp": + null, "userAltSecId": null, "userIss": "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", + "userTenantId": "00000000-0000-0000-0000-000000000000", "userName": "Nikolay + Rovinskiy", "upn": "username@microsoft.com"}, "userId": "00000000-0000-0000-0000-000000000000", + "token": "eyJhbGciOiJSUzI1NiIsImtpZCI6Ijk3QTcyRUQyOUNFMjMwMTQwQjVGNzFEOTkyODk2NzBDRDRGNEJFMzUiLCJ0eXAiOiJKV1QifQ.eyJyb2xlIjoiQ29udHJpYnV0b3IiLCJzY29wZSI6Ii9zdWJzY3JpcHRpb25zLzZhNmZmZjAwLTQ0NjQtNGVhYi1hNmIxLTBiNTMzYzcyMDJlMC9yZXNvdXJjZUdyb3Vwcy9uaXJvdmlucy1yZy1lYXN0dXMvcHJvdmlkZXJzL01pY3Jvc29mdC5NYWNoaW5lTGVhcm5pbmdTZXJ2aWNlcy93b3Jrc3BhY2VzL25pcm92aW5zLXdzLWVhc3R1cyIsImFjY291bnRpZCI6IjAwMDAwMDAwLTAwMDAtMDAwMC0wMDAwLTAwMDAwMDAwMDAwMCIsIndvcmtzcGFjZUlkIjoiYTEzOTFhZjMtOWYzOC00OGU4LTg2MmYtZGIzODQ2ZTgxZGQ0IiwicHJvamVjdGlkIjoiMDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAwIiwiZGlzY292ZXJ5IjoidXJpOi8vZGlzY292ZXJ5dXJpLyIsInRpZCI6IjcyZjk4OGJmLTg2ZjEtNDFhZi05MWFiLTJkN2NkMDExZGI0NyIsIm9pZCI6Ijc4NzViYWE2LTAyOGQtNDljOC1iZDRkLWE3NzJhYTRkYzAxMyIsInB1aWQiOiIxMDAzQkZGREE4QTREMEU3IiwiaXNzIjoiYXp1cmVtbCIsImFwcGlkIjoiTmlrb2xheSBSb3ZpbnNraXkiLCJleHAiOjE3MTk1NTUwNDUsImF1ZCI6ImF6dXJlbWwifQ.kaxuNrkEz4ARPy6HrRPEacX_xSZtHq4yBavn8TzVcP5UgaOCPpMPDWP20kddt2heommexMarDJ22MNzCB_S4x6ZbAfbg8TqBc7eohTq67Dirl2G5hfU0LEbgGxBPsaHSgdwPcVpnV6BGb1EfFctKJk9AegfY9hBNnZjDWJxTwdDxkNnM0PrwZAHqNqEyDPCVe2BGeogIN2n9IVPf80uMkcpTJQr_SyV1KV6uhUS-8975Xj9-wK9-1_XVBOJQcWX3-kkRRl-dfGFk59w7eFJ7eku6h9mm0G7Ft2qzMiR4Zmu3yG8T5cCQYDX8k2OaxIhHqJi1awjux_BvYJuwoXfg5A", + "tokenExpiryTimeUtc": "2024-06-28T06:10:45.8846867+00:00", "error": null, + "warnings": null, "revision": 2, "statusRevision": 0, "runUuid": "cef277fa-67fc-497e-aac1-aab3810fcd5d", + "parentRunUuid": null, "rootRunUuid": "cef277fa-67fc-497e-aac1-aab3810fcd5d", + "lastStartTimeUtc": "2024-06-07T05:05:06.998+00:00", "currentComputeTime": + "00:00:00", "computeDuration": null, "effectiveStartTimeUtc": "2024-06-07T05:05:06.998+00:00", + "lastModifiedBy": {"userObjectId": "00000000-0000-0000-0000-000000000000", + "userPuId": "1003BFFDA8A4D0E7", "userIdp": null, "userAltSecId": null, "userIss": + "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", "userTenantId": + "00000000-0000-0000-0000-000000000000", "userName": "Nikolay Rovinskiy", "upn": + "username@microsoft.com"}, "lastModifiedUtc": "2024-06-07T05:05:15.7040489+00:00", + "duration": null, "cancelationReason": null, "currentAttemptId": 1, "runId": + "feb068c0-fea0-4f0f-b5d0-b2a8960bd97b", "parentRunId": null, "experimentId": + "00000000-0000-0000-0000-000000000000", "status": "Running", "startTimeUtc": + "2024-06-07T05:05:06.998+00:00", "endTimeUtc": null, "scheduleId": null, "displayName": + "green_island_wpnhqk6x", "name": null, "dataContainerId": "dcid.feb068c0-fea0-4f0f-b5d0-b2a8960bd97b", + "description": null, "hidden": false, "runType": null, "runTypeV2": {"orchestrator": + null, "traits": ["mlflow"], "attribution": null, "computeType": null}, "properties": + {"mlflow.artifactUri": "azureml://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/experiments/0/runs/feb068c0-fea0-4f0f-b5d0-b2a8960bd97b/artifacts", + "_azureml.evaluation_run": "azure-ai-generative-parent", "_azureml.evaluate_artifacts": + "[{\"path\": \"eval_results.jsonl\", \"type\": \"table\"}]", "isEvaluatorRun": + "true"}, "parameters": {}, "actionUris": {}, "scriptName": null, "target": + null, "uniqueChildRunComputeTargets": [], "tags": {"mlflow.user": "promptflow-evals"}, + "settings": {}, "services": {}, "inputDatasets": [], "outputDatasets": [], + "runDefinition": null, "jobSpecification": null, "primaryMetricName": null, + "createdFrom": null, "cancelUri": null, "completeUri": null, "diagnosticsUri": + null, "computeRequest": null, "compute": null, "retainForLifetimeOfWorkspace": + false, "queueingInfo": null, "inputs": null, "outputs": null}' + headers: + connection: + - keep-alive + content-length: + - '4412' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.115' + status: + code: 200 + message: OK +- request: + body: '{"run_uuid": "feb068c0-fea0-4f0f-b5d0-b2a8960bd97b", "key": "f1.f1_score", + "value": 0.0, "timestamp": "1717563256242", "step": 0, "run_id": "feb068c0-fea0-4f0f-b5d0-b2a8960bd97b"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '177' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api/2.0/mlflow/runs/log-metric + response: + body: + string: '{}' + headers: + connection: + - keep-alive + content-length: + - '3' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-request-time: + - '0.660' + status: + code: 200 + message: OK +- request: + body: '{"run_uuid": "feb068c0-fea0-4f0f-b5d0-b2a8960bd97b", "status": "FINISHED", + "end_time": "1717563261483", "run_id": "feb068c0-fea0-4f0f-b5d0-b2a8960bd97b"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '151' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api/2.0/mlflow/runs/update + response: + body: + string: '{"run_info": {"run_uuid": "feb068c0-fea0-4f0f-b5d0-b2a8960bd97b", "experiment_id": + "5ea666e7-ae55-4060-a8a3-71e6e03fab9c", "run_name": "green_island_wpnhqk6x", + "user_id": "00000000-0000-0000-0000-000000000000", "status": "FINISHED", "start_time": + "1717736706998", "end_time": "1717736721378", "lifecycle_stage": "active", + "run_id": "feb068c0-fea0-4f0f-b5d0-b2a8960bd97b"}}' + headers: + connection: + - keep-alive + content-length: + - '376' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.368' + status: + code: 200 + message: OK +- request: + body: '{"runId": "evals_e2etests_target_fn_wqo0_peh_20240606_102622_386974", "selectRunMetadata": + true, "selectRunDefinition": true, "selectJobSpecification": true}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '115' + Content-Type: + - application/json + User-Agent: + - python-requests/2.31.0 + method: POST + uri: https://eastus2.api.azureml.ms/history/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/rundata + response: + body: + string: '{"error": {"code": "UserError", "severity": null, "message": "Run runId=eval_test_run2 + was not found", "messageFormat": "Run {runId} was not found", "messageParameters": + {"runId": "runId=eval_test_run2"}, "referenceCode": null, "detailsUri": null, + "target": null, "details": [], "innerError": {"code": "NotFoundError", "innerError": + null}, "debugInfo": null, "additionalInfo": null}, "correlation": {"operation": + "53e7a6173f3c96e20095cf51e078267e", "request": "31f90e833dfccf2a"}, "environment": + "eastus2", "location": "eastus2", "time": "2024-06-07T05:04:47.7973049+00:00", + "componentName": "run-history", "statusCode": 404}' + headers: + connection: + - keep-alive + content-length: + - '735' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.151' + status: + code: 404 + message: Run runId=eval_test_run2 was not found +version: 1 diff --git a/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_log_artifact.yaml b/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_log_artifact.yaml new file mode 100644 index 00000000000..dda5c1e1645 --- /dev/null +++ b/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_log_artifact.yaml @@ -0,0 +1,286 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/0.0.1.dev0 azure-ai-ml/1.10.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus2", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus2.api.azureml.ms/discovery"}}' + headers: + cache-control: + - no-cache + content-length: + - '2895' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.024' + status: + code: 200 + message: OK +- request: + body: '{"experiment_id": "0", "user_id": "promptflow-evals", "start_time": "1717563256142", + "tags": [{"key": "mlflow.user", "value": "promptflow-evals"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '145' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api/2.0/mlflow/runs/create + response: + body: + string: '{"run": {"info": {"run_uuid": "7f83a0e0-368b-40d9-9cc8-20d2459a3422", + "experiment_id": "5ea666e7-ae55-4060-a8a3-71e6e03fab9c", "run_name": "maroon_spade_wxz8fvsh", + "user_id": "00000000-0000-0000-0000-000000000000", "status": "RUNNING", "start_time": + "1717630666967", "artifact_uri": "azureml://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/experiments/0/runs/7f83a0e0-368b-40d9-9cc8-20d2459a3422/artifacts", + "lifecycle_stage": "active", "run_id": "7f83a0e0-368b-40d9-9cc8-20d2459a3422"}, + "data": {"tags": [{"key": "mlflow.user", "value": "promptflow-evals"}, {"key": + "mlflow.rootRunId", "value": "7f83a0e0-368b-40d9-9cc8-20d2459a3422"}, {"key": + "mlflow.runName", "value": "maroon_spade_wxz8fvsh"}, {"key": "mlflow.user", + "value": "Nikolay Rovinskiy"}]}, "inputs": {}}}' + headers: + connection: + - keep-alive + content-length: + - '935' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.126' + status: + code: 200 + message: OK +- request: + body: '{"paths": [{"path": "test_log_artifact0/test.json"}, {"path": "test_log_artifact0/internal_dir/test.json"}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '108' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/history/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/experimentids/5ea666e7-ae55-4060-a8a3-71e6e03fab9c/runs/7f83a0e0-368b-40d9-9cc8-20d2459a3422/artifacts/batch/metadata + response: + body: + string: '{"artifacts": {"test_log_artifact0/test.json": {"artifactId": "ExperimentRun/dcid.7f83a0e0-368b-40d9-9cc8-20d2459a3422/test_log_artifact0/test.json", + "origin": "ExperimentRun", "container": "dcid.7f83a0e0-368b-40d9-9cc8-20d2459a3422", + "path": "test_log_artifact0/test.json", "etag": null, "createdTime": "2024-06-05T23:37:51.8839305+00:00", + "dataPath": null, "tags": {}}, "test_log_artifact0/internal_dir/test.json": + {"artifactId": "ExperimentRun/dcid.7f83a0e0-368b-40d9-9cc8-20d2459a3422/test_log_artifact0/internal_dir/test.json", + "origin": "ExperimentRun", "container": "dcid.7f83a0e0-368b-40d9-9cc8-20d2459a3422", + "path": "test_log_artifact0/internal_dir/test.json", "etag": null, "createdTime": + "2024-06-05T23:37:51.8839523+00:00", "dataPath": null, "tags": {}}}, "artifactContentInformation": + {"test_log_artifact0/test.json": {"contentUri": "https://nirovinswseast5028951403.blob.core.windows.net/azureml/ExperimentRun/dcid.7f83a0e0-368b-40d9-9cc8-20d2459a3422/test_log_artifact0/test.json?sv=2019-07-07&sr=b&sig=tfmX8Yz1RFlg7uwEIBaRjwUWiNgKor%2FxepKJLkS2mNE%3D&skoid=f4777969-da99-42e7-a95c-5f6396e15e83&sktid=00000000-0000-0000-0000-000000000000&skt=2024-06-05T23%3A27%3A51Z&ske=2024-06-07T07%3A37%3A51Z&sks=b&skv=2019-07-07&st=2024-06-05T23%3A27%3A51Z&se=2024-06-06T23%3A37%3A51Z&sp=rcw", + "origin": "ExperimentRun", "container": "dcid.7f83a0e0-368b-40d9-9cc8-20d2459a3422", + "path": "test_log_artifact0/test.json", "tags": {}}, "test_log_artifact0/internal_dir/test.json": + {"contentUri": "https://nirovinswseast5028951403.blob.core.windows.net/azureml/ExperimentRun/dcid.7f83a0e0-368b-40d9-9cc8-20d2459a3422/test_log_artifact0/internal_dir/test.json?sv=2019-07-07&sr=b&sig=fR1tOSMpWHXIpiyFs7e4F4g3VxOMYTg4q23pxfvv%2B%2FA%3D&skoid=f4777969-da99-42e7-a95c-5f6396e15e83&sktid=00000000-0000-0000-0000-000000000000&skt=2024-06-05T23%3A27%3A51Z&ske=2024-06-07T07%3A37%3A51Z&sks=b&skv=2019-07-07&st=2024-06-05T23%3A27%3A51Z&se=2024-06-06T23%3A37%3A51Z&sp=rcw", + "origin": "ExperimentRun", "container": "dcid.7f83a0e0-368b-40d9-9cc8-20d2459a3422", + "path": "test_log_artifact0/internal_dir/test.json", "tags": {}}}, "errors": + {}}' + headers: + connection: + - keep-alive + content-length: + - '2339' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.460' + status: + code: 200 + message: OK +- request: + body: '{"f1": 0.5}' + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '11' + Content-Type: + - application/octet-stream + If-None-Match: + - '*' + User-Agent: + - azsdk-python-storage-blob/12.13.0 Python/3.11.5 (Windows-10-10.0.22631-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Wed, 05 Jun 2024 23:37:52 GMT + x-ms-version: + - '2021-08-06' + method: PUT + uri: https://nirovinswseast5028951403.blob.core.windows.net/azureml/ExperimentRun/dcid.7f83a0e0-368b-40d9-9cc8-20d2459a3422/test_log_artifact0/test.json?se=2024-06-06T23%3A37%3A51Z&sig=tfmX8Yz1RFlg7uwEIBaRjwUWiNgKor%2FxepKJLkS2mNE%3D&ske=2024-06-07T07%3A37%3A51Z&skoid=f4777969-da99-42e7-a95c-5f6396e15e83&sks=b&skt=2024-06-05T23%3A27%3A51Z&sktid=00000000-0000-0000-0000-000000000000&skv=2019-07-07&sp=rcw&sr=b&st=2024-06-05T23%3A27%3A51Z&sv=2019-07-07 + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - diUhBBdPqiF5HmMm8IgKdw== + last-modified: + - Wed, 05 Jun 2024 23:37:52 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - 5eDdDLCk6vg= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2021-08-06' + status: + code: 201 + message: Created +- request: + body: '{"internal_f1": 0.6}' + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '20' + Content-Type: + - application/octet-stream + If-None-Match: + - '*' + User-Agent: + - azsdk-python-storage-blob/12.13.0 Python/3.11.5 (Windows-10-10.0.22631-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Wed, 05 Jun 2024 23:37:53 GMT + x-ms-version: + - '2021-08-06' + method: PUT + uri: https://nirovinswseast5028951403.blob.core.windows.net/azureml/ExperimentRun/dcid.7f83a0e0-368b-40d9-9cc8-20d2459a3422/test_log_artifact0/internal_dir/test.json?se=2024-06-06T23%3A37%3A51Z&sig=fR1tOSMpWHXIpiyFs7e4F4g3VxOMYTg4q23pxfvv%2B%2FA%3D&ske=2024-06-07T07%3A37%3A51Z&skoid=f4777969-da99-42e7-a95c-5f6396e15e83&sks=b&skt=2024-06-05T23%3A27%3A51Z&sktid=00000000-0000-0000-0000-000000000000&skv=2019-07-07&sp=rcw&sr=b&st=2024-06-05T23%3A27%3A51Z&sv=2019-07-07 + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - ajrc77H5/pKmiiAzHB+q7A== + last-modified: + - Wed, 05 Jun 2024 23:37:53 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - +o3ucAmEJMI= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2021-08-06' + status: + code: 201 + message: Created +- request: + body: '{"run_uuid": "7f83a0e0-368b-40d9-9cc8-20d2459a3422", "status": "FINISHED", + "end_time": "1717563261483", "run_id": "7f83a0e0-368b-40d9-9cc8-20d2459a3422"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '151' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api/2.0/mlflow/runs/update + response: + body: + string: '{"run_info": {"run_uuid": "7f83a0e0-368b-40d9-9cc8-20d2459a3422", "experiment_id": + "5ea666e7-ae55-4060-a8a3-71e6e03fab9c", "run_name": "maroon_spade_wxz8fvsh", + "user_id": "00000000-0000-0000-0000-000000000000", "status": "FINISHED", "start_time": + "1717630666967", "end_time": "1717630676075", "lifecycle_stage": "active", + "run_id": "7f83a0e0-368b-40d9-9cc8-20d2459a3422"}}' + headers: + connection: + - keep-alive + content-length: + - '376' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.473' + status: + code: 200 + message: OK +version: 1 diff --git a/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_logging_metrics.yaml b/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_logging_metrics.yaml new file mode 100644 index 00000000000..7aa082ae2cd --- /dev/null +++ b/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_logging_metrics.yaml @@ -0,0 +1,177 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/0.0.1.dev0 azure-ai-ml/1.10.0 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus2", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus2.api.azureml.ms/discovery"}}' + headers: + cache-control: + - no-cache + content-length: + - '2895' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.024' + status: + code: 200 + message: OK +- request: + body: '{"experiment_id": "0", "user_id": "promptflow-evals", "start_time": "1717563256142", + "tags": [{"key": "mlflow.user", "value": "promptflow-evals"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '145' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api/2.0/mlflow/runs/create + response: + body: + string: '{"run": {"info": {"run_uuid": "e3f73360-3987-4947-9eef-1229059f5574", + "experiment_id": "5ea666e7-ae55-4060-a8a3-71e6e03fab9c", "run_name": "epic_seed_9jt0rqzb", + "user_id": "00000000-0000-0000-0000-000000000000", "status": "RUNNING", "start_time": + "1717632174490", "artifact_uri": "azureml://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/experiments/0/runs/e3f73360-3987-4947-9eef-1229059f5574/artifacts", + "lifecycle_stage": "active", "run_id": "e3f73360-3987-4947-9eef-1229059f5574"}, + "data": {"tags": [{"key": "mlflow.user", "value": "promptflow-evals"}, {"key": + "mlflow.rootRunId", "value": "e3f73360-3987-4947-9eef-1229059f5574"}, {"key": + "mlflow.runName", "value": "epic_seed_9jt0rqzb"}, {"key": "mlflow.user", "value": + "Nikolay Rovinskiy"}]}, "inputs": {}}}' + headers: + connection: + - keep-alive + content-length: + - '929' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.551' + status: + code: 200 + message: OK +- request: + body: '{"run_uuid": "e3f73360-3987-4947-9eef-1229059f5574", "key": "f1", "value": + 0.54, "timestamp": "1717563256242", "step": 0, "run_id": "e3f73360-3987-4947-9eef-1229059f5574"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '169' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api/2.0/mlflow/runs/log-metric + response: + body: + string: '{}' + headers: + connection: + - keep-alive + content-length: + - '3' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-content-type-options: + - nosniff + x-request-time: + - '0.454' + status: + code: 200 + message: OK +- request: + body: '{"run_uuid": "e3f73360-3987-4947-9eef-1229059f5574", "status": "FINISHED", + "end_time": "1717563261483", "run_id": "e3f73360-3987-4947-9eef-1229059f5574"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '151' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api/2.0/mlflow/runs/update + response: + body: + string: '{"run_info": {"run_uuid": "e3f73360-3987-4947-9eef-1229059f5574", "experiment_id": + "5ea666e7-ae55-4060-a8a3-71e6e03fab9c", "run_name": "epic_seed_9jt0rqzb", + "user_id": "00000000-0000-0000-0000-000000000000", "status": "FINISHED", "start_time": + "1717632174490", "end_time": "1717632182757", "lifecycle_stage": "active", + "run_id": "e3f73360-3987-4947-9eef-1229059f5574"}}' + headers: + connection: + - keep-alive + content-length: + - '373' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.578' + status: + code: 200 + message: OK +version: 1 diff --git a/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_writing_to_run_history.yaml b/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_writing_to_run_history.yaml new file mode 100644 index 00000000000..3709fb4e0c0 --- /dev/null +++ b/src/promptflow-evals/tests/recordings/azure/test_metrics_upload_TestMetricsUpload_test_writing_to_run_history.yaml @@ -0,0 +1,212 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - promptflow-azure-sdk/0.0.1.dev0 azure-ai-ml/1.16.1 azsdk-python-mgmt-machinelearningservices/0.1.0 + Python/3.11.5 (Windows-10-10.0.22631-SP0) + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000 + response: + body: + string: '{"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000", + "name": "00000", "type": "Microsoft.MachineLearningServices/workspaces", "location": + "eastus2", "tags": {}, "etag": null, "kind": "Default", "sku": {"name": "Basic", + "tier": "Basic"}, "properties": {"discoveryUrl": "https://eastus2.api.azureml.ms/discovery"}}' + headers: + cache-control: + - no-cache + content-length: + - '2978' + content-type: + - application/json; charset=utf-8 + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-request-time: + - '0.028' + status: + code: 200 + message: OK +- request: + body: '{"experiment_id": "0", "user_id": "promptflow-evals", "start_time": "1717563256142", + "tags": [{"key": "mlflow.user", "value": "promptflow-evals"}]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '145' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api/2.0/mlflow/runs/create + response: + body: + string: '{"run": {"info": {"run_uuid": "e4102cfe-0237-4993-905b-0aa0f3909df1", + "experiment_id": "5ea666e7-ae55-4060-a8a3-71e6e03fab9c", "run_name": "modest_bucket_xwbqj7lt", + "user_id": "00000000-0000-0000-0000-000000000000", "status": "RUNNING", "start_time": + "1717777037652", "artifact_uri": "azureml://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/experiments/0/runs/e4102cfe-0237-4993-905b-0aa0f3909df1/artifacts", + "lifecycle_stage": "active", "run_id": "e4102cfe-0237-4993-905b-0aa0f3909df1"}, + "data": {"tags": [{"key": "mlflow.user", "value": "promptflow-evals"}, {"key": + "mlflow.rootRunId", "value": "e4102cfe-0237-4993-905b-0aa0f3909df1"}, {"key": + "mlflow.runName", "value": "modest_bucket_xwbqj7lt"}, {"key": "mlflow.user", + "value": "Nikolay Rovinskiy"}]}, "inputs": {}}}' + headers: + connection: + - keep-alive + content-length: + - '937' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.604' + status: + code: 200 + message: OK +- request: + body: '{"runId": "evals_e2etests_target_fn_wqo0_peh_20240606_102622_386974", "properties": + {"test": 42}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '77' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: PATCH + uri: https://eastus2.api.azureml.ms/history/v1.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/experimentids/5ea666e7-ae55-4060-a8a3-71e6e03fab9c/runs/e4102cfe-0237-4993-905b-0aa0f3909df1 + response: + body: + string: '{"runNumber": 1717777040, "rootRunId": "e4102cfe-0237-4993-905b-0aa0f3909df1", + "createdUtc": "2024-06-07T16:17:17.652+00:00", "createdBy": {"userObjectId": + "00000000-0000-0000-0000-000000000000", "userPuId": "1003BFFDA8A4D0E7", "userIdp": + null, "userAltSecId": null, "userIss": "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", + "userTenantId": "00000000-0000-0000-0000-000000000000", "userName": "Nikolay + Rovinskiy", "upn": "username@microsoft.com"}, "userId": "00000000-0000-0000-0000-000000000000", + "token": "eyJhbGciOiJSUzI1NiIsImtpZCI6Ijk3QTcyRUQyOUNFMjMwMTQwQjVGNzFEOTkyODk2NzBDRDRGNEJFMzUiLCJ0eXAiOiJKV1QifQ.eyJyb2xlIjoiQ29udHJpYnV0b3IiLCJzY29wZSI6Ii9zdWJzY3JpcHRpb25zLzZhNmZmZjAwLTQ0NjQtNGVhYi1hNmIxLTBiNTMzYzcyMDJlMC9yZXNvdXJjZUdyb3Vwcy9uaXJvdmlucy1yZy1lYXN0dXMvcHJvdmlkZXJzL01pY3Jvc29mdC5NYWNoaW5lTGVhcm5pbmdTZXJ2aWNlcy93b3Jrc3BhY2VzL25pcm92aW5zLXdzLWVhc3R1cyIsImFjY291bnRpZCI6IjAwMDAwMDAwLTAwMDAtMDAwMC0wMDAwLTAwMDAwMDAwMDAwMCIsIndvcmtzcGFjZUlkIjoiYTEzOTFhZjMtOWYzOC00OGU4LTg2MmYtZGIzODQ2ZTgxZGQ0IiwicHJvamVjdGlkIjoiMDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAwIiwiZGlzY292ZXJ5IjoidXJpOi8vZGlzY292ZXJ5dXJpLyIsInRpZCI6IjcyZjk4OGJmLTg2ZjEtNDFhZi05MWFiLTJkN2NkMDExZGI0NyIsIm9pZCI6Ijc4NzViYWE2LTAyOGQtNDljOC1iZDRkLWE3NzJhYTRkYzAxMyIsInB1aWQiOiIxMDAzQkZGREE4QTREMEU3IiwiaXNzIjoiYXp1cmVtbCIsImFwcGlkIjoiTmlrb2xheSBSb3ZpbnNraXkiLCJleHAiOjE3MTk1OTg2NDAsImF1ZCI6ImF6dXJlbWwifQ.Q548NMQZczwBKrVO7A57_WfPNd0LQ7nw2fG8rpZ7Yf3LO6UEfEqo8P8BoeInFxFhX4_6ciX9AiLuxChlgqnVrR4kj9GyRt0eRRMyP4DOKKB1LDFeQOf-t59pYHwdYrpzoOlovHBQm86pV4GG__49qxB1JEsqh87YqnU0cijDZf2LjAL_iU5jCzyAyNeJdhIbcvlllMYRYAifj-IrLwcLITqOrUrrlOpJ8QXz6adLLh8PACfO1O2iO87KqjtFAKS93V0ekcsAfj3af3Qlw7B2cEOckRfnPov602GwdxFlJU40f5hFz1P1GEF5TyKzNNLOazQQwZ40rYSQOCXBxxuycA", + "tokenExpiryTimeUtc": "2024-06-28T18:17:20.9502572+00:00", "error": null, + "warnings": null, "revision": 2, "statusRevision": 0, "runUuid": "33ab0848-e4f0-4a96-96ad-4492c80bfe1b", + "parentRunUuid": null, "rootRunUuid": "33ab0848-e4f0-4a96-96ad-4492c80bfe1b", + "lastStartTimeUtc": "2024-06-07T16:17:17.652+00:00", "currentComputeTime": + "00:00:00", "computeDuration": null, "effectiveStartTimeUtc": "2024-06-07T16:17:17.652+00:00", + "lastModifiedBy": {"userObjectId": "00000000-0000-0000-0000-000000000000", + "userPuId": "1003BFFDA8A4D0E7", "userIdp": null, "userAltSecId": null, "userIss": + "https://sts.windows.net/00000000-0000-0000-0000-000000000000/", "userTenantId": + "00000000-0000-0000-0000-000000000000", "userName": "Nikolay Rovinskiy", "upn": + "username@microsoft.com"}, "lastModifiedUtc": "2024-06-07T16:17:23.350837+00:00", + "duration": null, "cancelationReason": null, "currentAttemptId": 1, "runId": + "e4102cfe-0237-4993-905b-0aa0f3909df1", "parentRunId": null, "experimentId": + "00000000-0000-0000-0000-000000000000", "status": "Running", "startTimeUtc": + "2024-06-07T16:17:17.652+00:00", "endTimeUtc": null, "scheduleId": null, "displayName": + "modest_bucket_xwbqj7lt", "name": null, "dataContainerId": "dcid.e4102cfe-0237-4993-905b-0aa0f3909df1", + "description": null, "hidden": false, "runType": null, "runTypeV2": {"orchestrator": + null, "traits": ["mlflow"], "attribution": null, "computeType": null}, "properties": + {"mlflow.artifactUri": "azureml://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/experiments/0/runs/e4102cfe-0237-4993-905b-0aa0f3909df1/artifacts", + "test": "42"}, "parameters": {}, "actionUris": {}, "scriptName": null, "target": + null, "uniqueChildRunComputeTargets": [], "tags": {"mlflow.user": "promptflow-evals"}, + "settings": {}, "services": {}, "inputDatasets": [], "outputDatasets": [], + "runDefinition": null, "jobSpecification": null, "primaryMetricName": null, + "createdFrom": null, "cancelUri": null, "completeUri": null, "diagnosticsUri": + null, "computeRequest": null, "compute": null, "retainForLifetimeOfWorkspace": + false, "queueingInfo": null, "inputs": null, "outputs": null}' + headers: + connection: + - keep-alive + content-length: + - '4243' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.036' + status: + code: 200 + message: OK +- request: + body: '{"run_uuid": "e4102cfe-0237-4993-905b-0aa0f3909df1", "status": "FINISHED", + "end_time": "1717563261483", "run_id": "e4102cfe-0237-4993-905b-0aa0f3909df1"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '151' + Content-Type: + - application/json + User-Agent: + - promptflow/0.0.1.dev0 + method: POST + uri: https://eastus2.api.azureml.ms/mlflow/v2.0/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/api/2.0/mlflow/runs/update + response: + body: + string: '{"run_info": {"run_uuid": "e4102cfe-0237-4993-905b-0aa0f3909df1", "experiment_id": + "5ea666e7-ae55-4060-a8a3-71e6e03fab9c", "run_name": "modest_bucket_xwbqj7lt", + "user_id": "00000000-0000-0000-0000-000000000000", "status": "FINISHED", "start_time": + "1717777037652", "end_time": "1717777046571", "lifecycle_stage": "active", + "run_id": "e4102cfe-0237-4993-905b-0aa0f3909df1"}}' + headers: + connection: + - keep-alive + content-length: + - '377' + content-type: + - application/json; charset=utf-8 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-request-time: + - '0.493' + status: + code: 200 + message: OK +version: 1 diff --git a/src/promptflow-recording/promptflow/recording/azure/bases.py b/src/promptflow-recording/promptflow/recording/azure/bases.py index 2760ec575f1..ddaa59fd15c 100644 --- a/src/promptflow-recording/promptflow/recording/azure/bases.py +++ b/src/promptflow-recording/promptflow/recording/azure/bases.py @@ -6,11 +6,12 @@ import inspect import json from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional import vcr from vcr import matchers from vcr.request import Request +from vcr.util import read_body from ..record_mode import is_live, is_record, is_replay from .constants import FILTER_HEADERS, TEST_CLASSES_FOR_RUN_INTEGRATION_TEST_RECORDING, SanitizedValues @@ -71,6 +72,7 @@ def from_test_case(test_class, test_func_name: str, **kwargs) -> "PFAzureIntegra user_object_id=kwargs["user_object_id"], tenant_id=kwargs["tenant_id"], variable_recorder=kwargs["variable_recorder"], + recording_dir=kwargs.get("recording_dir"), ) else: return PFAzureIntegrationTestRecording( @@ -287,8 +289,8 @@ def _custom_request_path_matcher(self, r1: Request, r2: Request) -> bool: if "https://" in r1.path: _path = str(r1.path) endpoint = ".blob.core.windows.net/" - duplicate_path = _path[_path.index(endpoint) + len(endpoint) :] - path_for_compare = _path[: _path.index("https://")] + duplicate_path[duplicate_path.index("/") + 1 :] + duplicate_path = _path[_path.index(endpoint) + len(endpoint):] + path_for_compare = _path[: _path.index("https://")] + duplicate_path[duplicate_path.index("/") + 1:] return path_for_compare == r2.path # for blob storage request, sanitize the upload hash in path if r1.host == r2.host and r1.host == SanitizedValues.BLOB_STORAGE_REQUEST_HOST: @@ -333,7 +335,7 @@ def _custom_request_body_matcher(self, r1: Request, r2: Request) -> bool: body2_dict = json.loads(r2.body.decode("utf-8")) body_dict["startTimeUtc"] = body2_dict["startTimeUtc"] body_dict["endTimeUtc"] = body2_dict["endTimeUtc"] - except (AttributeError, json.JSONDecodeError, KeyError): + except (AttributeError, json.JSONDecodeError, KeyError, TypeError): return False body1 = json.dumps(body_dict) _r1.body = body1.encode("utf-8") @@ -346,6 +348,39 @@ def _custom_request_body_matcher(self, r1: Request, r2: Request) -> bool: # so simple return True for such requests # corresponding test: `test_upload_run` if r1.method == "PUT": - if "PromptFlowArtifacts/batch_run_name" in r1.path or "ExperimentRun/dcid.batch_run_name" in r1.path: + if ("PromptFlowArtifacts/batch_run_name" in r1.path or + "ExperimentRun/dcid.batch_run_name" in r1.path or + 'executionlogs.txt' in r1.path or + ("PromptFlowArtifacts" in r1.path and "flow_artifacts" in r1.path)): return True - return matchers.body(r1, r2) + return self._body(r1, r2) + + def _transform_bytes_may_be(self, byte_obj: Optional[bytes]) -> Optional[bytes]: + """ + Try to fix line ending. + + :param byte_obj: The original bytes object. + :type byte_obj: Optional[bytes] + :return: byte_obj with fixed end lines. + """ + if byte_obj is None: + return byte_obj + return byte_obj.replace(b'\r\n', b'\n') + + def _body(self, r1: Request, r2: Request) -> None: + """ + Implementation of body matcher. + + If recordigs are generated on one system and it is replayed on another one, + line ending may not match. For this purpose we replace \r\n by \n. + :param r1: The request. + :type r1: Request + :param r2: Another request. + :type r2: Request + :raises AssertionError: if bodies does not match. + """ + b1 = self._transform_bytes_may_be(read_body(r1)) + b2 = self._transform_bytes_may_be(read_body(r2)) + + if b1 != b2: + raise AssertionError diff --git a/src/promptflow-recording/promptflow/recording/azure/constants.py b/src/promptflow-recording/promptflow/recording/azure/constants.py index ec1a45335e5..0d85142f813 100644 --- a/src/promptflow-recording/promptflow/recording/azure/constants.py +++ b/src/promptflow-recording/promptflow/recording/azure/constants.py @@ -64,6 +64,17 @@ class SanitizedValues: USERNAME = "unknown_user" # MISC EMAIL_USERNAME = "username" + # run start and end time + START_TIME = "1717563256142" + TIMESTAMP = "1717563256242" + END_TIME = "1717563261483" + # Promptflow RunID + RUN_ID = "evals_e2etests_target_fn_wqo0_peh_20240606_102622_386974" + # Fake Application insights event + FAKE_APP_INSIGHTS = [ + {"ver": 1, "name": "Microsoft.ApplicationInsights.Event", + "time": "2024-06-06T23:20:59.838896Z", "sampleRate": 100.0, + "iKey": UUID, "tags": {"foo": "bar"}}] class AzureMLResourceTypes: @@ -79,4 +90,5 @@ class AzureMLResourceTypes: "TestTelemetry", "TestAzureCliPerf", "TestCSharpSdk", + "TestMetricsUpload", ] diff --git a/src/promptflow-recording/promptflow/recording/azure/utils.py b/src/promptflow-recording/promptflow/recording/azure/utils.py index cf3bcf3728c..5253260ca7b 100644 --- a/src/promptflow-recording/promptflow/recording/azure/utils.py +++ b/src/promptflow-recording/promptflow/recording/azure/utils.py @@ -200,6 +200,19 @@ def sanitize_pfs_request_body(body: str) -> str: # PFS will help handle this field, so client does not need to pass this value if "runExperimentName" in body: body_dict["runExperimentName"] = "" + # Start and end time + if "start_time" in body_dict: + body_dict["start_time"] = SanitizedValues.START_TIME + if "timestamp" in body_dict: + body_dict["timestamp"] = SanitizedValues.TIMESTAMP + if "end_time" in body_dict: + body_dict["end_time"] = SanitizedValues.END_TIME + # Promptflow Run ID + if "runId" in body_dict: + body_dict["runId"] = SanitizedValues.RUN_ID + # Sanitize telemetry event + if isinstance(body_dict, list) and "Microsoft.ApplicationInsights.Event" in body: + body_dict = SanitizedValues.FAKE_APP_INSIGHTS return json.dumps(body_dict)