From f6099c4f1bb73efb958efc36232f1ba9a2f2a13b Mon Sep 17 00:00:00 2001 From: Rhyann Clarke <146747548+rclarke0@users.noreply.github.com> Date: Thu, 25 Apr 2024 08:48:42 -0400 Subject: [PATCH] Correct HS Time Calculation and Print Statements (#15005) # Overview Concise print statements and HS on time calculation correction. # Test Plan Tested on ABR robots. # Changelog Previously, the heater shaker temperature on time was linked to the shaker on time. This has been corrected to reference the correct command string. If the heatershaker is not deactivated, the on time is calculated with the protocol end time stamp. Changed print statements for get robot logs and abr_google_drive to make it more obvious to the user if there is an error or not. # Review requests # Risk assessment --- .../automation/google_drive_tool.py | 11 +++-- .../automation/google_sheets_tool.py | 7 +++- .../data_collection/abr_calibration_logs.py | 34 +++++++-------- .../data_collection/get_run_logs.py | 11 ++--- .../data_collection/read_robot_logs.py | 41 +++++++++++++------ 5 files changed, 64 insertions(+), 40 deletions(-) diff --git a/abr-testing/abr_testing/automation/google_drive_tool.py b/abr-testing/abr_testing/automation/google_drive_tool.py index 3b65456d0ff..44ec6a68f27 100644 --- a/abr-testing/abr_testing/automation/google_drive_tool.py +++ b/abr-testing/abr_testing/automation/google_drive_tool.py @@ -4,6 +4,7 @@ import webbrowser import mimetypes from oauth2client.service_account import ServiceAccountCredentials # type: ignore[import] +import googleapiclient # type: ignore[import] from googleapiclient.discovery import build from googleapiclient.http import MediaFileUpload @@ -58,7 +59,6 @@ def list_folder(self, delete: Any = False) -> Set[str]: break if not file_names: print("No folders or files found in Google Drive.") - print(f"{len(file_names)} item(s) in Google Drive") return file_names def delete_files(self, file_or_folder_id: str) -> None: @@ -98,18 +98,22 @@ def upload_missing_files(self, storage_directory: str) -> None: file for file in os.listdir(storage_directory) if file.endswith(".json") ) missing_files = local_files_json - set(google_drive_files_json) - print(f"Missing files: {len(missing_files)}") # Upload missing files. uploaded_files = [] for file in missing_files: file_path = os.path.join(storage_directory, file) uploaded_file_id = google_drive.upload_file(self, file_path) - self.share_permissions(uploaded_file_id) uploaded_files.append( {"name": os.path.basename(file_path), "id": uploaded_file_id} ) + try: + self.share_permissions(uploaded_file_id) + except googleapiclient.errors.HttpError: + continue + # Fetch the updated file list after all files are uploaded files = google_drive.list_folder(self) + file_names = [file for file in files] for uploaded_file in uploaded_files: this_name = uploaded_file["name"] @@ -121,6 +125,7 @@ def upload_missing_files(self, storage_directory: str) -> None: print( f"File '{this_name}' was not found in the list of files after uploading." ) + print(f"{len(files)} item(s) in Google Drive") def open_folder(self) -> Optional[str]: """Open folder in web browser.""" diff --git a/abr-testing/abr_testing/automation/google_sheets_tool.py b/abr-testing/abr_testing/automation/google_sheets_tool.py index af38a39dcc0..0fcc104fe1e 100644 --- a/abr-testing/abr_testing/automation/google_sheets_tool.py +++ b/abr-testing/abr_testing/automation/google_sheets_tool.py @@ -2,6 +2,7 @@ import gspread # type: ignore[import] import socket import httplib2 +import time as t from datetime import datetime from oauth2client.service_account import ServiceAccountCredentials # type: ignore[import] from typing import Dict, List, Any, Set, Tuple @@ -71,6 +72,10 @@ def write_to_row(self, data: List) -> None: print("UNABLE TO CONNECT TO SERVER!!, CHECK CONNECTION") except Exception as error: print(error.__traceback__) + except gspread.exceptions.APIError: + print("Write quotes exceeded. Waiting 30 sec before writing.") + t.sleep(30) + self.worksheet.insert_row(data, index=self.row_index) def delete_row(self, row_index: int) -> None: """Delete Row from google sheet.""" @@ -94,7 +99,7 @@ def get_column(self, column_number: int) -> Set[str]: def get_index_row(self) -> int: """Check for the next available row to write too.""" row_index = len(self.get_column(1)) - print("Row Index: ", row_index) + print(f"Row Index: {row_index} recorded on google sheet.") return row_index def update_row_index(self) -> None: diff --git a/abr-testing/abr_testing/data_collection/abr_calibration_logs.py b/abr-testing/abr_testing/data_collection/abr_calibration_logs.py index 4d744b5b2f5..c0833ff6c8c 100644 --- a/abr-testing/abr_testing/data_collection/abr_calibration_logs.py +++ b/abr-testing/abr_testing/data_collection/abr_calibration_logs.py @@ -5,6 +5,7 @@ import json import gspread # type: ignore[import] import sys +import time as t from abr_testing.data_collection import read_robot_logs from abr_testing.automation import google_drive_tool, google_sheets_tool @@ -18,16 +19,20 @@ def check_for_duplicates( headers: List[str], ) -> Union[List[str], None]: """Check google sheet for duplicates.""" + t.sleep(5) serials = google_sheet.get_column(col_1) modify_dates = google_sheet.get_column(col_2) - # check for complete calibration. - if len(row[-1]) > 0: - for serial, modify_date in zip(serials, modify_dates): - if row[col_1 - 1] == serial and row[col_2 - 1] == modify_date: - print(f"Skipped row for instrument {serial}. Already on Google Sheet.") - return None - read_robot_logs.write_to_sheets(sheet_location, google_sheet, row, headers) - print(f"Writing calibration for: {serial}") + # Check for calibration time stamp. + if row[-1] is not None: + if len(row[-1]) > 0: + for serial, modify_date in zip(serials, modify_dates): + if row[col_1 - 1] == serial and row[col_2 - 1] == modify_date: + print( + f"Skipped row for instrument {serial}. Already on Google Sheet." + ) + return None + read_robot_logs.write_to_sheets(sheet_location, google_sheet, row, headers) + print(f"Writing calibration for: {row[7]}") return row @@ -206,15 +211,10 @@ def upload_calibration_offsets( if ip_or_all == "ALL": ip_address_list = ip_file["ip_address_list"] for ip in ip_address_list: - print(ip) - try: - saved_file_path, calibration = read_robot_logs.get_calibration_offsets( - ip, storage_directory - ) - upload_calibration_offsets(calibration, storage_directory) - except Exception: - print(f"ERROR: Failed to read IP address: {ip}") - continue + saved_file_path, calibration = read_robot_logs.get_calibration_offsets( + ip, storage_directory + ) + upload_calibration_offsets(calibration, storage_directory) else: saved_file_path, calibration = read_robot_logs.get_calibration_offsets( ip_or_all, storage_directory diff --git a/abr-testing/abr_testing/data_collection/get_run_logs.py b/abr-testing/abr_testing/data_collection/get_run_logs.py index 4034f076dc9..d8f60afbf8e 100644 --- a/abr-testing/abr_testing/data_collection/get_run_logs.py +++ b/abr-testing/abr_testing/data_collection/get_run_logs.py @@ -104,13 +104,10 @@ def get_all_run_logs(storage_directory: str) -> None: ip_address_list = ip_file["ip_address_list"] runs_from_storage = read_robot_logs.get_run_ids_from_google_drive(google_drive) for ip in ip_address_list: - try: - runs = get_run_ids_from_robot(ip) - runs_to_save = read_robot_logs.get_unseen_run_ids(runs, runs_from_storage) - save_runs(runs_to_save, ip, storage_directory) - google_drive.upload_missing_files(storage_directory) - except Exception: - print(f"ERROR: Failed to read IP address: {ip}.") + runs = get_run_ids_from_robot(ip) + runs_to_save = read_robot_logs.get_unseen_run_ids(runs, runs_from_storage) + save_runs(runs_to_save, ip, storage_directory) + google_drive.upload_missing_files(storage_directory) if __name__ == "__main__": diff --git a/abr-testing/abr_testing/data_collection/read_robot_logs.py b/abr-testing/abr_testing/data_collection/read_robot_logs.py index 48ef1d20163..dc8c9b32c45 100644 --- a/abr-testing/abr_testing/data_collection/read_robot_logs.py +++ b/abr-testing/abr_testing/data_collection/read_robot_logs.py @@ -12,6 +12,7 @@ import time as t import json import requests +import sys def lpc_data(file_results: Dict[str, Any], protocol_info: Dict) -> List[Dict[str, Any]]: @@ -72,9 +73,10 @@ def hs_commands(file_results: Dict[str, Any]) -> Dict[str, float]: hs_home_count: float = 0.0 hs_speed: float = 0.0 hs_rotations: Dict[str, float] = dict() - hs_temps: Dict[str, float] = dict() + hs_temps: Dict[float, float] = dict() temp_time = None shake_time = None + deactivate_time = None for command in commandData: commandType = command["commandType"] # Heatershaker @@ -87,17 +89,21 @@ def hs_commands(file_results: Dict[str, Any]) -> Dict[str, float]: # Home count elif commandType == "heaterShaker/deactivateShaker": hs_home_count += 1 + shake_deactivate_time = datetime.strptime( + command.get("startedAt", ""), "%Y-%m-%dT%H:%M:%S.%f%z" + ) + if shake_time is not None and shake_deactivate_time > shake_time: + shake_duration = (shake_deactivate_time - shake_time).total_seconds() + hs_rotations[hs_speed] = hs_rotations.get(hs_speed, 0.0) + ( + (hs_speed * shake_duration) / 60 + ) + elif commandType == "heaterShaker/deactivateHeater": deactivate_time = datetime.strptime( command.get("startedAt", ""), "%Y-%m-%dT%H:%M:%S.%f%z" ) if temp_time is not None and deactivate_time > temp_time: temp_duration = (deactivate_time - temp_time).total_seconds() hs_temps[hs_temp] = hs_temps.get(hs_temp, 0.0) + temp_duration - if shake_time is not None and deactivate_time > shake_time: - shake_duration = (deactivate_time - shake_time).total_seconds() - hs_rotations[hs_speed] = hs_rotations.get(hs_speed, 0.0) + ( - (hs_speed * shake_duration) / 60 - ) # of Rotations elif commandType == "heaterShaker/setAndWaitForShakeSpeed": hs_speed = command["params"]["rpm"] @@ -111,6 +117,13 @@ def hs_commands(file_results: Dict[str, Any]) -> Dict[str, float]: temp_time = datetime.strptime( command.get("completedAt", ""), "%Y-%m-%dT%H:%M:%S.%f%z" ) + if temp_time is not None and deactivate_time is None: + # If heater shaker module is not deactivated, protocol completedAt time stamp used. + protocol_end = datetime.strptime( + file_results.get("completedAt", ""), "%Y-%m-%dT%H:%M:%S.%f%z" + ) + temp_duration = (protocol_end - temp_time).total_seconds() + hs_temps[hs_temp] = hs_temps.get(hs_temp, 0.0) + temp_duration hs_latch_sets = hs_latch_count / 2 # one set of open/close hs_total_rotations = sum(hs_rotations.values()) hs_total_temp_time = sum(hs_temps.values()) @@ -254,7 +267,7 @@ def create_abr_data_sheet( file_name_csv = file_name + ".csv" sheet_location = os.path.join(storage_directory, file_name_csv) if os.path.exists(sheet_location): - print(f"File {sheet_location} located. Not overwriting.") + return sheet_location else: with open(sheet_location, "w") as csvfile: writer = csv.DictWriter(csvfile, fieldnames=headers) @@ -368,7 +381,6 @@ def get_run_ids_from_storage(storage_directory: str) -> Set[str]: def get_unseen_run_ids(runs: Set[str], runs_from_storage: Set[str]) -> Set[str]: """Subtracts runs from storage from current runs being read.""" runs_to_save = runs - runs_from_storage - print(f"There are {str(len(runs_to_save))} new run(s) to save.") return runs_to_save @@ -406,7 +418,7 @@ def write_to_sheets( google_sheet.write_header(headers) google_sheet.update_row_index() google_sheet.write_to_row(row_list) - t.sleep(5) # Sleep added to avoid API error. + t.sleep(5) def get_calibration_offsets( @@ -415,9 +427,14 @@ def get_calibration_offsets( """Connect to robot via ip and get calibration data.""" calibration = dict() # Robot Information [Name, Software Version] - response = requests.get( - f"http://{ip}:31950/health", headers={"opentrons-version": "3"} - ) + try: + response = requests.get( + f"http://{ip}:31950/health", headers={"opentrons-version": "3"} + ) + print(f"Connected to {ip}") + except Exception: + print(f"ERROR: Failed to read IP address: {ip}") + sys.exit() health_data = response.json() robot_name = health_data.get("name", "") api_version = health_data.get("api_version", "")