diff --git a/ScenarioMaker/tab.py b/ScenarioMaker/tab.py index f57d7c43..8a209de2 100644 --- a/ScenarioMaker/tab.py +++ b/ScenarioMaker/tab.py @@ -1126,6 +1126,8 @@ def save(self, copy_from_path=None): continue if file_extension == ".json": continue + if filename == "__pycache__": + continue shutil.copy(os.path.join(copy_from_path, fname), self.working_dir) # # Delete ScenarioMaker.json in save folder diff --git a/core/app_scenario.py b/core/app_scenario.py index 2b9c5f58..8c3197ce 100644 --- a/core/app_scenario.py +++ b/core/app_scenario.py @@ -3430,6 +3430,7 @@ def process_action(self, action, log_output=False): param = str(action['name']).strip("[]") inc_value = float(self._resolve_params_in_item(action['value'], component)) param_section, param_name = self._parse_param_name(param, component) + logging.debug(f"Incrementing parameter pre {param_section}:{param_name} to {inc_value}") param_value = float(Params.get(param_section, param_name)) new_value = str(param_value + inc_value) logging.debug(f"Incrementing parameter {param_section}:{param_name} to {new_value}") diff --git a/scenarios/macos/_library/Teams/teams_create_meeting/code_YTPC34.py b/scenarios/macos/_library/Teams/teams_create_meeting/code_YTPC34.py index 158aea94..0463d50d 100644 --- a/scenarios/macos/_library/Teams/teams_create_meeting/code_YTPC34.py +++ b/scenarios/macos/_library/Teams/teams_create_meeting/code_YTPC34.py @@ -32,28 +32,15 @@ def run(scenario): # logging.info("bots_test_server: " + bots_test_server) # logging.info("==========================================") - ##### - # Modification of these limits can result in loss of access to bot server - ##### - max_bots = 9 - max_duration = 600 - # Access key for VERY LIMITED INTERNAL testing of larger meetings. Do NOT use for normal testing. Large meetings from other access keys will be rejected and may result in loss of access to bot server. - if "dRXP2CvR58BXF2" in access_key or "WPYWEbsMfvoIPbX" in access_key: - max_bots = 49 - max_duration = 50000 - ##### + max_duration = 43200 # Validation of test params if access_key == "-1" and number_of_bots > 0: logging.error("No valid Teams Bots access key provided! Check that profile contains a Teams Bots Access key, or request one from HOBL Support (HOBLsupport@microsoft.com).") raise Exception("No Teams Bots Key Provided.") - if number_of_bots > max_bots: - logging.error("Requested bots exceeds the limit of " + str(max_bots) + ". Attempting to exceed these limits can result in permanent loss of access to bot services for your organization.") - raise Exception("Requested bots exceeds the limit of " + str(max_bots) + ". Attempting to exceed these limits can result in permanent loss of access to bot services for your organization.") if float(duration) > max_duration: logging.error("Requested call duration exceeds the limit of " + str(max_duration) + "s. Attempting to exceed these limits can result in permanent loss of access to bot services for your organization.") raise Exception("Requested call duration exceeds the limit of " + str(max_duration) + "s. Attempting to exceed these limits can result in permanent loss of access to bot services for your organization.") - if number_of_bots > 0: request_string = "" diff --git a/scenarios/windows/_library/Teams/teams_create_meeting/code_YTPC34.py b/scenarios/windows/_library/Teams/teams_create_meeting/code_YTPC34.py index b586d5e3..7deea3ca 100644 --- a/scenarios/windows/_library/Teams/teams_create_meeting/code_YTPC34.py +++ b/scenarios/windows/_library/Teams/teams_create_meeting/code_YTPC34.py @@ -32,28 +32,15 @@ def run(scenario): # logging.info("bots_test_server: " + bots_test_server) # logging.info("==========================================") - ##### - # Modification of these limits can result in loss of access to bot server - ##### - max_bots = 9 - max_duration = 600 - # Access key for VERY LIMITED INTERNAL testing of larger meetings. Do NOT use for normal testing. Large meetings from other access keys will be rejected and may result in loss of access to bot server. - if "dRXP2CvR58BXF2" in access_key or "WPYWEbsMfvoIPbX" in access_key: - max_bots = 49 - max_duration = 50000 - ##### + max_duration = 43200 # Validation of test params if access_key == "-1" and number_of_bots > 0: logging.error("No valid Teams Bots access key provided! Check that profile contains a Teams Bots Access key, or request one from HOBL Support (HOBLsupport@microsoft.com).") raise Exception("No Teams Bots Key Provided.") - if number_of_bots > max_bots: - logging.error("Requested bots exceeds the limit of " + str(max_bots) + ". Attempting to exceed these limits can result in permanent loss of access to bot services for your organization.") - raise Exception("Requested bots exceeds the limit of " + str(max_bots) + ". Attempting to exceed these limits can result in permanent loss of access to bot services for your organization.") if float(duration) > max_duration: logging.error("Requested call duration exceeds the limit of " + str(max_duration) + "s. Attempting to exceed these limits can result in permanent loss of access to bot services for your organization.") raise Exception("Requested call duration exceeds the limit of " + str(max_duration) + "s. Attempting to exceed these limits can result in permanent loss of access to bot services for your organization.") - if number_of_bots > 0: request_string = "" diff --git a/scenarios/windows/_library/procyon/procyon_run/__init__.py b/scenarios/windows/_library/procyon/procyon_run/__init__.py new file mode 100644 index 00000000..06f1dec9 --- /dev/null +++ b/scenarios/windows/_library/procyon/procyon_run/__init__.py @@ -0,0 +1 @@ +from .procyon_run import * \ No newline at end of file diff --git a/scenarios/windows/_library/procyon/procyon_run/code_1LC4L8F.py b/scenarios/windows/_library/procyon/procyon_run/code_1LC4L8F.py new file mode 100644 index 00000000..e5deed6b --- /dev/null +++ b/scenarios/windows/_library/procyon/procyon_run/code_1LC4L8F.py @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import logging +import csv +import os +import time +import math +import xml.etree.ElementTree as ET +from core.parameters import Params + +def run(scenario): + logging.debug('Executing code block: code_1LC4L8F.py') + test_def = Params.get('procyon_run', 'test_definition') + loop_num = Params.get('procyon_run', 'loop_number') + if loop_num is None or loop_num == "": + loop_num = "1" + loop_num = int(float(loop_num)) + if ".def" not in test_def: + test_def = test_def + ".def" + logging.info(f"Running Procyon loop {loop_num} using test definition: {test_def}") + scenario._remote_make_dir(f"{scenario.dut_data_path}\\procyon", delete=False) + scenario._call(["C:\\Program Files\\UL\\Procyon\\ProcyonCmd.exe", f"--definition={test_def} --loop=1 --export-xml=c:\\hobl_data\\procyon\\Results_{loop_num}.xml"], fail_on_exception=False) + + # Record timestamp + t = time.time() - scenario.scenario_start_time + # round up t to the nearest second + timestamp = math.ceil(t) + + # Download xml result file back to host + scenario._copy_data_from_remote(scenario.result_dir + "\\procyon", source=f"{scenario.dut_data_path}\\procyon") + + # Read scores from xml file and write to csv + xml_file = scenario.result_dir + f"\\procyon\\Results_{loop_num}.xml" + tree = ET.parse(xml_file) + root = tree.getroot() + + score_headers = [] + values = {} + for element in root.iter(): + if "Score" in element.tag: + if element.tag not in values: + score_headers.append(element.tag) + values[element.tag] = (element.text or "").strip() + + if not score_headers: + logging.warning(f"No score tags found in XML file: {xml_file}") + return + + headers = ["timestamp"] + score_headers + + csv_file = scenario.result_dir + f"\\procyon\\Procyon_Results.trace" + + csv_exists = os.path.exists(csv_file) + with open(csv_file, "a", newline="", encoding="utf-8") as file_handle: + writer = csv.writer(file_handle) + if not csv_exists: + writer.writerow(headers) + writer.writerow([timestamp] + [values.get(h, "") for h in score_headers]) + + # Read Procyon_results.trace file, average the scores for each column, and output into a summary csv as key,val pairs + summary_file = scenario.result_dir + f"\\Procyon_Summary.csv" + with open(csv_file, "r", newline="", encoding="utf-8") as file_handle: + reader = csv.DictReader(file_handle) + summary_data = {} + count = 0 + for row in reader: + count += 1 + for key, value in row.items(): + if key == "timestamp": + continue + try: + value = float(value) + except ValueError: + value = 0.0 + if key not in summary_data: + summary_data[key] = 0.0 + summary_data[key] += value + if count > 0: + for key in summary_data: + summary_data[key] /= count + + with open(summary_file, "w", newline="", encoding="utf-8") as file_handle: + writer = csv.writer(file_handle) + for key, value in summary_data.items(): + writer.writerow([f"Procyon-{key}", value]) diff --git a/scenarios/windows/_library/procyon/procyon_run/default_params.py b/scenarios/windows/_library/procyon/procyon_run/default_params.py new file mode 100644 index 00000000..63f8b86f --- /dev/null +++ b/scenarios/windows/_library/procyon/procyon_run/default_params.py @@ -0,0 +1,14 @@ +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +from core.parameters import Params +from utilities.open_source.modules import import_run_user_only + +def run(): + Params.setCalculated('scenario_section', __package__.split('.')[-1]) + run_user_only() + Params.setDefault('procyon_run', 'test_definition', 'office_productivity', desc='Workload to run', valOptions=['office_productivity', 'essentials', 'video_playback_batterylife', 'photo_editing', 'video_editing', 'ai_computer_vision_winml', 'ai_computer_vision_snpe', 'ai_computer_vision_openvino', 'ai_computer_vision_tensorrt', 'ai_computer_vision_ryzenai']) + return + +def run_user_only(): + return diff --git a/scenarios/windows/_library/procyon/procyon_run/procyon_run.json b/scenarios/windows/_library/procyon/procyon_run/procyon_run.json new file mode 100644 index 00000000..014b3652 --- /dev/null +++ b/scenarios/windows/_library/procyon/procyon_run/procyon_run.json @@ -0,0 +1,34 @@ +[ + { + "author": "jewilder", + "capture_device": "", + "children": [], + "create_date": "2026-05-06", + "description": "Run Procyon", + "enabled": true, + "id": "1LC3YWY", + "type": "Information" + }, + { + "children": [], + "delay": "0.0", + "description": "Workload to run", + "enabled": true, + "id": "1LC437A", + "name": "[test_definition]", + "type": "Set Default", + "val_options": "office_productivity,essentials,video_playback_batterylife,photo_editing,video_editing,ai_computer_vision_winml,ai_computer_vision_snpe,ai_computer_vision_openvino,ai_computer_vision_tensorrt,ai_computer_vision_ryzenai", + "value": "office_productivity" + }, + { + "children": [], + "delay": "0.0", + "description": "Run Procyon", + "enabled": true, + "file_name": [ + "code_1LC4L8F.py" + ], + "id": "1LC4L8F", + "type": "Code" + } +] \ No newline at end of file diff --git a/scenarios/windows/_library/procyon/procyon_run/procyon_run.py b/scenarios/windows/_library/procyon/procyon_run/procyon_run.py new file mode 100644 index 00000000..2403dd01 --- /dev/null +++ b/scenarios/windows/_library/procyon/procyon_run/procyon_run.py @@ -0,0 +1,59 @@ +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import core.app_scenario +from core.parameters import Params +import logging +import os +from . import default_params + +# Description: +# Automatically generated standard scenario. + +class ProcyonRun(core.app_scenario.Scenario): + # Set default parameters: + default_params.run() + + actions = None + + def setUp(self): + # Load actions JSON. + actions_json = os.path.join(os.path.dirname(__file__), "procyon_run.json") + self.actions = self.load_action_json(actions_json) + + # Execute Setup actions, if they exist + setup_action = self._find_next_type("Setup", json=self.actions) + if setup_action is not None: + self.run_actions(setup_action["children"]) + + # Call base class setUp() to dump config, call tool callbacks, and start measurment + core.app_scenario.Scenario.setUp(self) + + + def runTest(self): + # Execute Run Test actions, if they exist + runtest_action = self._find_next_type("Run Test", json=self.actions) + if runtest_action is not None: + self.run_actions(runtest_action["children"]) + return + + # If no "Run Test", "Setup", or "Teardown" specified, then just execute the whole list + setup_action = self._find_next_type("Setup", json=self.actions) + teardown_action = self._find_next_type("Teardown", json=self.actions) + if runtest_action is None and setup_action is None and teardown_action is None: + self.run_actions(self.actions) + + + def tearDown(self): + # Call base class tearDown() to stop measurment, copy back data from DUT, and call tool callbacks + core.app_scenario.Scenario.tearDown(self) + + # Execute Teardown actions, if they exist + teardown_action = self._find_next_type("Teardown", json=self.actions) + if teardown_action is not None: + self.run_actions(teardown_action["children"]) + + + def kill(self): + # In case of scenario failure or termination, kill any applications left open here: + return diff --git a/scenarios/windows/_library/procyon/procyon_setup/__init__.py b/scenarios/windows/_library/procyon/procyon_setup/__init__.py new file mode 100644 index 00000000..878b6b1a --- /dev/null +++ b/scenarios/windows/_library/procyon/procyon_setup/__init__.py @@ -0,0 +1 @@ +from .procyon_setup import * \ No newline at end of file diff --git a/scenarios/windows/_library/procyon/procyon_setup/code_1LCYJ1C.py b/scenarios/windows/_library/procyon/procyon_setup/code_1LCYJ1C.py new file mode 100644 index 00000000..0a499670 --- /dev/null +++ b/scenarios/windows/_library/procyon/procyon_setup/code_1LCYJ1C.py @@ -0,0 +1,39 @@ +import logging +import time +from core.parameters import Params + +def run(scenario): + logging.debug('Executing code block: code_1L9EY7T.py') + logging.info("Preparing Procyon for first use.") + + # Get and verify parameters + host_path = Params.get('procyon_setup', 'host_path') + if host_path == "" or host_path is None: + logging.warning("Procyon host_path is not set, not installing.") + return + license_key = Params.get('procyon_setup', 'key') + if license_key == "": + logging.error("Procyon license key is not set.") + scenario.fail("Procyon license key is not set.") + + # Check if already installed + if not scenario.checkPrepStatusNew([("procyon", [host_path])]): + logging.info("Procyon already installed.") + return + + # Upload installer + target = f"{scenario.dut_exec_path}\\procyon_setup" + logging.info(f"Uploading Procyon setup files to {target}") + scenario._upload(host_path + "\\*", target, check_modified=False) + + # Run installer + logging.info("Running Procyon setup script.") + scenario._call([f"{target}\\procyon-setup.exe", "/s /sms"], expected_exit_code="0") + + # Register license + logging.info("Registering license.") + scenario._call(["C:\\Program Files\\UL\\Procyon\\ProcyonCmd.exe", f"--register={license_key}"]) + + # Create prep status file to indicate completion + scenario.createPrepStatusControlFile(suffix=[host_path], module="procyon") + scenario._sleep_to_now() diff --git a/scenarios/windows/_library/procyon/procyon_setup/default_params.py b/scenarios/windows/_library/procyon/procyon_setup/default_params.py new file mode 100644 index 00000000..61ac952a --- /dev/null +++ b/scenarios/windows/_library/procyon/procyon_setup/default_params.py @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +from core.parameters import Params +from utilities.open_source.modules import import_run_user_only + +def run(): + Params.setCalculated('scenario_section', __package__.split('.')[-1]) + run_user_only() + Params.setDefault('procyon_setup', 'host_path', '', desc='Path on host to Procyon setup files', valOptions=[]) + Params.setDefault('procyon_setup', 'key', '', desc='Procyon license key', valOptions=[]) + return + +def run_user_only(): + return diff --git a/scenarios/windows/_library/procyon/procyon_setup/procyon_setup.json b/scenarios/windows/_library/procyon/procyon_setup/procyon_setup.json new file mode 100644 index 00000000..7f083c24 --- /dev/null +++ b/scenarios/windows/_library/procyon/procyon_setup/procyon_setup.json @@ -0,0 +1,45 @@ +[ + { + "author": "jewilder", + "capture_device": "", + "children": [], + "create_date": "2026-05-07", + "description": "Setup Procyon", + "enabled": true, + "id": "1LCYHU8", + "type": "Information" + }, + { + "children": [], + "delay": "0.0", + "description": "Path on host to Procyon setup files", + "enabled": true, + "id": "1LE65M3", + "name": "[host_path]", + "type": "Set Default", + "val_options": "", + "value": "" + }, + { + "children": [], + "delay": "0.0", + "description": "Procyon license key", + "enabled": true, + "id": "1LE6615", + "name": "[key]", + "type": "Set Default", + "val_options": "", + "value": "" + }, + { + "children": [], + "delay": "0.0", + "description": "Install Procyon", + "enabled": true, + "file_name": [ + "code_1LCYJ1C.py" + ], + "id": "1LCYJ1C", + "type": "Code" + } +] \ No newline at end of file diff --git a/scenarios/windows/_library/procyon/procyon_setup/procyon_setup.py b/scenarios/windows/_library/procyon/procyon_setup/procyon_setup.py new file mode 100644 index 00000000..e02affc6 --- /dev/null +++ b/scenarios/windows/_library/procyon/procyon_setup/procyon_setup.py @@ -0,0 +1,59 @@ +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import core.app_scenario +from core.parameters import Params +import logging +import os +from . import default_params + +# Description: +# Automatically generated standard scenario. + +class ProcyonSetup(core.app_scenario.Scenario): + # Set default parameters: + default_params.run() + + actions = None + + def setUp(self): + # Load actions JSON. + actions_json = os.path.join(os.path.dirname(__file__), "procyon_setup.json") + self.actions = self.load_action_json(actions_json) + + # Execute Setup actions, if they exist + setup_action = self._find_next_type("Setup", json=self.actions) + if setup_action is not None: + self.run_actions(setup_action["children"]) + + # Call base class setUp() to dump config, call tool callbacks, and start measurment + core.app_scenario.Scenario.setUp(self) + + + def runTest(self): + # Execute Run Test actions, if they exist + runtest_action = self._find_next_type("Run Test", json=self.actions) + if runtest_action is not None: + self.run_actions(runtest_action["children"]) + return + + # If no "Run Test", "Setup", or "Teardown" specified, then just execute the whole list + setup_action = self._find_next_type("Setup", json=self.actions) + teardown_action = self._find_next_type("Teardown", json=self.actions) + if runtest_action is None and setup_action is None and teardown_action is None: + self.run_actions(self.actions) + + + def tearDown(self): + # Call base class tearDown() to stop measurment, copy back data from DUT, and call tool callbacks + core.app_scenario.Scenario.tearDown(self) + + # Execute Teardown actions, if they exist + teardown_action = self._find_next_type("Teardown", json=self.actions) + if teardown_action is not None: + self.run_actions(teardown_action["children"]) + + + def kill(self): + # In case of scenario failure or termination, kill any applications left open here: + return diff --git a/scenarios/windows/lvp/lvp.py b/scenarios/windows/lvp/lvp.py index 4d0680b0..0d307f53 100644 --- a/scenarios/windows/lvp/lvp.py +++ b/scenarios/windows/lvp/lvp.py @@ -175,7 +175,7 @@ def playMovie(self): f"//*[contains(@Name, 'Full screen') and @ClassName = 'AppBarButton']" ).click() - ActionChains(self.driver).move_by_offset(-200, -200).click().perform() + ActionChains(self.driver).move_by_offset(-200, -200).perform() except: # Navigate to Personal menu self.driver.find_element_by_name("Personal").click() @@ -204,7 +204,7 @@ def playMovie(self): # Move mouse away from the menu time.sleep(5) - ActionChains(self.driver).move_by_offset(150, -150).click().perform() + ActionChains(self.driver).move_by_offset(150, -150).perform() def tearDown(self): logging.info("Performing teardown.") diff --git a/scenarios/windows/procyon/__init__.py b/scenarios/windows/procyon/__init__.py new file mode 100644 index 00000000..43053efa --- /dev/null +++ b/scenarios/windows/procyon/__init__.py @@ -0,0 +1 @@ +from .procyon import * \ No newline at end of file diff --git a/scenarios/windows/procyon/default_params.py b/scenarios/windows/procyon/default_params.py new file mode 100644 index 00000000..876de8fa --- /dev/null +++ b/scenarios/windows/procyon/default_params.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +from core.parameters import Params +from utilities.open_source.modules import import_run_user_only + +def run(): + Params.setCalculated('scenario_section', __package__.split('.')[-1]) + run_user_only() + Params.setDefault('procyon', 'test', 'office_productivity', desc='', valOptions=['office_productivity', 'essentials', 'video_playback_batterylife', 'photo_editing', 'video_editing', 'ai_computer_vision_winml', 'ai_computer_vision_snpe', 'ai_computer_vision_openvino', 'ai_computer_vision_tensorrt', 'ai_computer_vision_ryzenai']) + Params.setDefault('procyon', 'loops', '1', desc='Number of time to iterate the benchmark', valOptions=[]) + Params.setDefault('procyon', 'key', '', desc='Procyon activation key to be used during setup', valOptions=[]) + Params.setDefault('procyon', 'host_path', '', desc='Path on host to Procyon setup files', valOptions=[]) + return + +def run_user_only(): + import_run_user_only('scenarios\\windows\\_library\\procyon\\procyon_run') + import_run_user_only('scenarios\\windows\\_library\\procyon\\procyon_setup') + return diff --git a/scenarios/windows/procyon/procyon.json b/scenarios/windows/procyon/procyon.json new file mode 100644 index 00000000..527803d4 --- /dev/null +++ b/scenarios/windows/procyon/procyon.json @@ -0,0 +1,153 @@ +[ + { + "author": "jewilder", + "capture_device": "", + "children": [], + "create_date": "2026-05-07", + "description": "Procyon", + "enabled": true, + "id": "1LCXHN7", + "type": "Information" + }, + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "1LCXMCT", + "name": "[test]", + "type": "Set Default", + "val_options": "office_productivity,essentials,video_playback_batterylife,photo_editing,video_editing,ai_computer_vision_winml,ai_computer_vision_snpe,ai_computer_vision_openvino,ai_computer_vision_tensorrt,ai_computer_vision_ryzenai", + "value": "office_productivity" + }, + { + "children": [], + "delay": "0.0", + "description": "Number of time to iterate the benchmark", + "enabled": true, + "id": "1LCXM0J", + "name": "[loops]", + "type": "Set Default", + "val_options": "", + "value": "1" + }, + { + "children": [], + "delay": "0.0", + "description": "Procyon activation key to be used during setup", + "enabled": true, + "id": "1LCXNX5", + "name": "[key]", + "type": "Set Default", + "val_options": "", + "value": "" + }, + { + "children": [], + "delay": "0.0", + "description": "Path on host to Procyon setup files", + "enabled": true, + "id": "1LE67UW", + "name": "[host_path]", + "type": "Set Default", + "val_options": "", + "value": "" + }, + { + "children": [ + { + "children": [], + "delay": "0", + "description": "Procyon setup", + "enabled": true, + "id": "1LE67A6", + "include_path": "scenarios\\windows\\_library\\procyon\\procyon_setup", + "params": [ + { + "name": "[host_path]", + "val_options": "", + "value": "[host_path]" + }, + { + "name": "[key]", + "val_options": "", + "value": "[key]" + } + ], + "type": "Include" + } + ], + "description": "", + "enabled": true, + "id": "1LCXJ2W", + "type": "Setup" + }, + { + "children": [ + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "1LE6NL4", + "name": "[procyon_run:loop_number]", + "type": "Set", + "value": "1" + }, + { + "children": [ + { + "children": [], + "delay": "0", + "description": "Run Procyon", + "enabled": true, + "id": "1LEPNM3", + "include_path": "scenarios\\windows\\_library\\procyon\\procyon_run", + "params": [ + { + "name": "[test_definition]", + "val_options": "office_productivity,video_playback_batterylife,photo_editing,video_editing,ai_computer_vision_winml,ai_computer_vision_snpe,ai_computer_vision_openvino,ai_computer_vision_tensorrt", + "value": "[test]" + } + ], + "type": "Include" + }, + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "1LE6P1K", + "name": "[procyon_run:loop_number]", + "type": "Increment", + "value": "1" + } + ], + "count": "[loops]", + "delay": "0.0", + "description": "", + "enabled": true, + "id": "1LE6MPR", + "type": "Loop" + }, + { + "children": [], + "description": "", + "enabled": true, + "id": "1LE6MPT", + "type": "End Loop" + } + ], + "description": "", + "enabled": true, + "id": "1LCXJFT", + "type": "Run Test" + }, + { + "children": [], + "description": "", + "enabled": true, + "id": "1LCXJK2", + "type": "Teardown" + } +] \ No newline at end of file diff --git a/scenarios/windows/procyon/procyon.py b/scenarios/windows/procyon/procyon.py new file mode 100644 index 00000000..406b3b2e --- /dev/null +++ b/scenarios/windows/procyon/procyon.py @@ -0,0 +1,67 @@ +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import core.app_scenario +from core.parameters import Params +import logging +import os +from . import default_params + +# Description: +# Automatically generated standard scenario. + +class Procyon(core.app_scenario.Scenario): + # Set default parameters: + default_params.run() + + actions = None + + def setUp(self): + # Load actions JSON. + actions_json = os.path.join(os.path.dirname(__file__), "procyon.json") + self.actions = self.load_action_json(actions_json) + + # Execute Setup actions, if they exist + setup_action = self._find_next_type("Setup", json=self.actions) + if setup_action is not None: + self.run_actions(setup_action["children"]) + + # Call base class setUp() to dump config, call tool callbacks, and start measurment + core.app_scenario.Scenario.setUp(self) + + + def runTest(self): + # Execute Run Test actions, if they exist + runtest_action = self._find_next_type("Run Test", json=self.actions) + if runtest_action is not None: + self.run_actions(runtest_action["children"]) + return + + # If no "Run Test", "Setup", or "Teardown" specified, then just execute the whole list + setup_action = self._find_next_type("Setup", json=self.actions) + teardown_action = self._find_next_type("Teardown", json=self.actions) + if runtest_action is None and setup_action is None and teardown_action is None: + self.run_actions(self.actions) + + + def tearDown(self): + # Call base class tearDown() to stop measurment, copy back data from DUT, and call tool callbacks + core.app_scenario.Scenario.tearDown(self) + + # Execute Teardown actions, if they exist + teardown_action = self._find_next_type("Teardown", json=self.actions) + if teardown_action is not None: + self.run_actions(teardown_action["children"]) + + + def kill(self): + # In case of scenario failure or termination, kill any applications left open here: + try: + self._kill("procyon.exe") + except: + pass + + try: + self._kill("Excel.exe Powerpnt.exe Winword.exe") + except: + pass diff --git a/scenarios/windows/teams_procyon/__init__.py b/scenarios/windows/teams_procyon/__init__.py new file mode 100644 index 00000000..aab7caaa --- /dev/null +++ b/scenarios/windows/teams_procyon/__init__.py @@ -0,0 +1 @@ +from .teams_procyon import * \ No newline at end of file diff --git a/scenarios/windows/teams_procyon/default_params.py b/scenarios/windows/teams_procyon/default_params.py new file mode 100644 index 00000000..b3e9baae --- /dev/null +++ b/scenarios/windows/teams_procyon/default_params.py @@ -0,0 +1,34 @@ +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +from core.parameters import Params +from utilities.open_source.modules import import_run_user_only + +def run(): + Params.setCalculated('scenario_section', __package__.split('.')[-1]) + run_user_only() + Params.setDefault('procyon', 'test', 'office_productivity', desc='Workload to run', valOptions=['office_productivity', 'essentials', 'video_playback_batterylife', 'photo_editing', 'video_editing', 'ai_computer_vision_winml', 'ai_computer_vision_snpe', 'ai_computer_vision_openvino', 'ai_computer_vision_tensorrt', 'ai_computer_vision_ryzenai']) + Params.setDefault('procyon', 'warmup_delay', '1800', desc='Number of seconds to wait and let the device warm up before starting the Procyon run.', valOptions=[]) + Params.setDefault('procyon', 'host_path', '', desc='The path on the host where the Procyon installer has been downloaded.', valOptions=[]) + Params.setDefault('procyon', 'key', '', desc='Procyon license key', valOptions=[]) + Params.setParam('teams', 'send_screen', '1') + Params.setParam('teams', 'show_desktop', '1') + Params.setParam('teams', 'number_of_bots', '1') + Params.setParam('teams', 'send_video', '1') + Params.setParam('teams', 'send_audio', '1') + Params.setParam('teams', 'bots_send_video', '1') + Params.setParam('teams', 'bots_send_audio', '1') + Params.setParam('teams', 'bots_share_screen', '0') + Params.setParam('teams', 'bots_force_subscribe_resolution', '0') + Params.setParam(None, 'phase_reporting', '1') + return + +def run_user_only(): + import_run_user_only('scenarios\\windows\\_library\\Teams\\teams_setup') + import_run_user_only('scenarios\\windows\\_library\\Teams\\teams_teardown') + import_run_user_only('scenarios\\windows\\_library\\procyon\\procyon_run') + import_run_user_only('scenarios\\windows\\_library\\procyon\\procyon_setup') + import_run_user_only('scenarios\\windows\\_library\\run_command') + import_run_user_only('scenarios\\windows\\_library\\window_move') + Params.setUserDefault('teams', 'duration', '3600', desc='Sets the time in seconds for the test to run.', valOptions=['60', '120', '240', '300', '600', '900']) + return diff --git a/scenarios/windows/teams_procyon/teams_procyon.json b/scenarios/windows/teams_procyon/teams_procyon.json new file mode 100644 index 00000000..de2d2fc0 --- /dev/null +++ b/scenarios/windows/teams_procyon/teams_procyon.json @@ -0,0 +1,339 @@ +[ + { + "author": "", + "capture_device": "", + "children": [], + "create_date": "", + "description": "Runs Teams and Procyon", + "enabled": true, + "id": "YTN8NX", + "type": "Information" + }, + { + "children": [], + "delay": "0.0", + "description": "Set Default Params", + "enabled": true, + "id": "YTNUEV", + "type": "Comment" + }, + { + "children": [], + "delay": "0.0", + "description": "Workload to run", + "enabled": true, + "id": "1HV5R1F", + "name": "[procyon:test]", + "type": "Set Default", + "val_options": "office_productivity,essentials,video_playback_batterylife,photo_editing,video_editing,ai_computer_vision_winml,ai_computer_vision_snpe,ai_computer_vision_openvino,ai_computer_vision_tensorrt,ai_computer_vision_ryzenai", + "value": "office_productivity" + }, + { + "children": [], + "delay": "0.0", + "description": "Number of seconds to wait and let the device warm up before starting the Procyon run.", + "enabled": true, + "id": "1HV5WL9", + "name": "[procyon:warmup_delay]", + "type": "Set Default", + "val_options": "", + "value": "1800" + }, + { + "children": [], + "delay": "0.0", + "description": "The path on the host where the Procyon installer has been downloaded.", + "enabled": true, + "id": "1L9EU5K", + "name": "[procyon:host_path]", + "type": "Set Default", + "val_options": "", + "value": "" + }, + { + "children": [], + "delay": "0.0", + "description": "Procyon license key", + "enabled": true, + "id": "1L9FP9K", + "name": "[procyon:key]", + "type": "Set Default", + "val_options": "", + "value": "" + }, + { + "children": [], + "delay": "0.0", + "description": "Sets the time in seconds for the test to run.", + "enabled": true, + "id": "1514842", + "multiple": false, + "name": "[teams:duration]", + "type": "Set User Default", + "val_options": "60,120,240,300,600,900", + "value": "3600" + }, + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "15VP7ER", + "name": "[teams:send_screen]", + "type": "Set", + "value": "1" + }, + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "15VP7MP", + "name": "[teams:show_desktop]", + "type": "Set", + "value": "1" + }, + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "15VP9C7", + "name": "[teams:number_of_bots]", + "type": "Set", + "value": "1" + }, + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "15VPALR", + "name": "[teams:send_video]", + "type": "Set", + "value": "1" + }, + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "162ATH5", + "name": "[teams:send_audio]", + "type": "Set", + "value": "1" + }, + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "15VPXTE", + "name": "[teams:bots_send_video]", + "type": "Set", + "value": "1" + }, + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "15VPXYE", + "name": "[teams:bots_send_audio]", + "type": "Set", + "value": "1" + }, + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "15VPY63", + "name": "[teams:bots_share_screen]", + "type": "Set", + "value": "0" + }, + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "162EFJK", + "name": "[teams:bots_force_subscribe_resolution]", + "type": "Set", + "value": "0" + }, + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "1FY2TRN", + "name": "[phase_reporting]", + "type": "Set", + "value": "1" + }, + { + "children": [ + { + "children": [], + "delay": "0", + "description": "", + "enabled": true, + "id": "1LFKLLC", + "include_path": "scenarios\\windows\\_library\\procyon\\procyon_setup", + "params": [ + { + "name": "[host_path]", + "val_options": "", + "value": "[procyon:host_path]" + }, + { + "name": "[key]", + "val_options": "", + "value": "[procyon:key]" + } + ], + "type": "Include" + }, + { + "children": [], + "delay": "0", + "description": "", + "enabled": true, + "id": "1H3CH75", + "include_path": "scenarios\\windows\\_library\\Teams\\teams_setup", + "type": "Include" + }, + { + "children": [], + "delay": "5", + "description": "", + "enabled": true, + "id": "1H39WNC", + "include_path": "scenarios\\windows\\_library\\run_command", + "params": [ + { + "name": "[command]", + "val_options": "", + "value": "[teams:join_meeting_uri]" + } + ], + "type": "Include" + }, + { + "children": [], + "delay": "0", + "description": "", + "enabled": true, + "id": "1H39Y9J", + "include_path": "scenarios\\windows\\_library\\window_move", + "params": [ + { + "name": "[screen]", + "val_options": "", + "value": "1" + } + ], + "type": "Include" + } + ], + "description": "", + "enabled": true, + "id": "YTXF95", + "type": "Setup" + }, + { + "children": [ + { + "children": [], + "delay": "[procyon:warmup_delay]", + "description": "Letting device warm up", + "enabled": true, + "id": "1LFKMUM", + "type": "Delay" + }, + { + "children": [], + "delay": "0", + "description": "", + "enabled": true, + "id": "1LFKM2W", + "include_path": "scenarios\\windows\\_library\\procyon\\procyon_run", + "params": [ + { + "name": "[test_definition]", + "val_options": "office_productivity,essentials,video_playback_batterylife,photo_editing,video_editing,ai_computer_vision_winml,ai_computer_vision_snpe,ai_computer_vision_openvino,ai_computer_vision_tensorrt,ai_computer_vision_ryzenai", + "value": "[procyon:test]" + } + ], + "type": "Include" + } + ], + "description": "", + "enabled": true, + "id": "YTX9MY", + "type": "Run Test" + }, + { + "children": [ + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "1H3A0RV", + "screen": "1", + "type": "Set Display" + }, + { + "children": [], + "delay": "2", + "description": "", + "enabled": true, + "id": "1H3AMUR", + "type": "Click Coord", + "x": "0.01", + "y": "0.01" + }, + { + "children": [], + "delay": "0", + "description": "", + "enabled": true, + "id": "1H3A14F", + "include_path": "scenarios\\windows\\_library\\window_move", + "params": [ + { + "name": "[screen]", + "val_options": "", + "value": "0" + } + ], + "type": "Include" + }, + { + "children": [], + "delay": "0.0", + "description": "", + "enabled": true, + "id": "1H3A197", + "screen": "0", + "type": "Set Display" + }, + { + "children": [], + "delay": "0", + "description": "", + "enabled": true, + "id": "13LPX71", + "include_path": "scenarios\\windows\\_library\\Teams\\teams_teardown", + "type": "Include" + } + ], + "description": "", + "enabled": true, + "id": "YTXARH", + "type": "Teardown" + } +] \ No newline at end of file diff --git a/scenarios/windows/teams_procyon/teams_procyon.py b/scenarios/windows/teams_procyon/teams_procyon.py new file mode 100644 index 00000000..728b7dde --- /dev/null +++ b/scenarios/windows/teams_procyon/teams_procyon.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import core.app_scenario +from core.parameters import Params +import logging +import os +from . import default_params + +# Description: +# Automatically generated standard scenario. + +class TeamsProcyon(core.app_scenario.Scenario): + module = __module__.split('.')[-1] + + prep_scenarios = ["teams_install", "office_install"] + + # Set default parameters: + default_params.run() + + actions = None + + def setUp(self): + # Load actions JSON. + actions_json = os.path.join(os.path.dirname(__file__), "teams_procyon.json") + self.actions = self.load_action_json(actions_json) + + # Execute Setup actions, if they exist + setup_action = self._find_next_type("Setup", json=self.actions) + if setup_action is not None: + self.run_actions(setup_action["children"]) + + # Call base class setUp() to dump config, call tool callbacks, and start measurment + core.app_scenario.Scenario.setUp(self) + + + def prep(self): + # if not self.checkPrepStatusNew([(self.module, self.prep_version)]): + # return + + logging.info("Preparing for first use.") + self.target = f"{self.dut_exec_path}\\Procyon" + + logging.info(f"Uploading Procyon files to {self.dut_exec_path}") + self._upload(self.resolve(f"scenarios\\windows\\{self.module}\\Procyon"), self.dut_exec_path) + + cmd = ("-ExecutionPolicy Bypass" f" -File {self.target}\\x86_Procyon_Setup.ps1") + + self._call(["powershell.exe", cmd], log_output=False, fail_on_exception=False, expected_exit_code="") + + self.createPrepStatusControlFile() + + + def runTest(self): + # Execute Run Test actions, if they exist + runtest_action = self._find_next_type("Run Test", json=self.actions) + if runtest_action is not None: + self.run_actions(runtest_action["children"]) + return + + # If no "Run Test", "Setup", or "Teardown" specified, then just execute the whole list + setup_action = self._find_next_type("Setup", json=self.actions) + teardown_action = self._find_next_type("Teardown", json=self.actions) + if runtest_action is None and setup_action is None and teardown_action is None: + self.run_actions(self.actions) + + + def tearDown(self): + # Call base class tearDown() to stop measurment, copy back data from DUT, and call tool callbacks + core.app_scenario.Scenario.tearDown(self) + + # Execute Teardown actions, if they exist + teardown_action = self._find_next_type("Teardown", json=self.actions) + if teardown_action is not None: + self.run_actions(teardown_action["children"]) + + + def kill(self): + self._call([ + "powershell.exe", + f"Get-CimInstance Win32_Process | Where-Object {{ $_.CommandLine -like '*Procyon*' }} | ForEach-Object {{ Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue }}" + ], expected_exit_code="") diff --git a/tools/screen_record.py b/tools/screen_record.py index fa1cfec5..51a1f5ab 100644 --- a/tools/screen_record.py +++ b/tools/screen_record.py @@ -49,7 +49,7 @@ def testBeginCallback(self): cmd = os.path.join(self.dut_exec_path, "ffmpeg.exe") args = "-f gdigrab -framerate 6 -i desktop -loglevel quiet -c:v libx264 -tune stillimage -crf 40 -pix_fmt yuv420p " + output_file stop_key = "q" - self._call(["powershell.exe", os.path.join(self.dut_exec_path, "command_wrapper.ps1"), " \"" + cmd + " \'" + args + "\' " + self.stop_file + " " + stop_key + "\"" ], blocking=False) + self._call(["powershell.exe", os.path.join(self.dut_exec_path, "command_wrapper.ps1") + " \"" + cmd + " \'" + args + "\' " + self.stop_file + " " + stop_key + "\"" ], blocking=False) elif self.platform.lower() == "macos": output_file = self.scenario.dut_data_path + "/" + self.scenario.testname + "_recording.mp4" cmd = "/opt/homebrew/bin/ffmpeg"