From 4791219dc014f8a300aa959aabae5753938be901 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Fri, 13 Jul 2018 11:15:09 -0700 Subject: [PATCH 1/3] Record release time data into bigquery --- .../compatibility_lib/compatibility_store.py | 47 +++++++++++++++++++ .../compatibility_lib/configs.py | 4 ++ 2 files changed, 51 insertions(+) diff --git a/compatibility_lib/compatibility_lib/compatibility_store.py b/compatibility_lib/compatibility_lib/compatibility_store.py index 1c268deb..464ce8be 100644 --- a/compatibility_lib/compatibility_lib/compatibility_store.py +++ b/compatibility_lib/compatibility_lib/compatibility_store.py @@ -22,11 +22,13 @@ from google.cloud import bigquery from google.cloud.bigquery import table +from compatibility_lib import configs from compatibility_lib import package _DATASET_NAME = 'compatibility_checker' _SELF_COMPATIBILITY_STATUS_TABLE_NAME = 'self_compatibility_status' _PAIRWISE_COMPATIBILITY_STATUS_TABLE_NAME = 'pairwise_compatibility_status' +_RELEASE_TIME_FOR_DEPENDENCIES_TABLE_NAME = 'release_time_for_dependencies' class Status(enum.Enum): @@ -136,6 +138,12 @@ def __init__(self): self._pairwise_table = self._client.get_table(dataset_ref.table( _PAIRWISE_COMPATIBILITY_STATUS_TABLE_NAME)) + self._release_time_table_id = ( + '{}.{}'.format( + _DATASET_NAME, _RELEASE_TIME_FOR_DEPENDENCIES_TABLE_NAME)) + self._release_time_table = self._client.get_table(dataset_ref.table( + _RELEASE_TIME_FOR_DEPENDENCIES_TABLE_NAME)) + @staticmethod def _row_to_compatibility_status(packages: Iterable[package.Package], row: table.Row) -> \ @@ -168,6 +176,28 @@ def _compatibility_status_to_row( row['install_name_lower'], row['install_name_higher'] = names return row + @staticmethod + def _compatibility_status_to_release_time_row( + cs: CompatibilityResult) -> List: + """Converts a CompatibilityResult into a dict which is a row for + release time table.""" + if len(cs.packages) != 1 or cs.dependency_info is None: + return [] + install_name = cs.packages[0].install_name + dependency_info = cs.dependency_info + rows = [] + + for pkg, version_info in dependency_info.items(): + row = { + 'install_name': install_name, + 'dep_name': pkg, + } + row.update(version_info) + row['timestamp'] = row.pop('current_time') + rows.append(row) + + return rows + @staticmethod def _filter_older_versions(crs: Iterable[CompatibilityResult]) \ -> Iterable[CompatibilityResult]: @@ -352,6 +382,7 @@ def save_compatibility_statuses( raise ValueError('CompatibilityResult must have 1 or 2 packages') rows = [self._compatibility_status_to_row(s) for s in compatibility_statuses] + self_rows = [r for r in rows if 'install_name' in r] pair_rows = [r for r in rows if 'install_name' not in r] if self_rows: @@ -362,3 +393,19 @@ def save_compatibility_statuses( self._client.insert_rows( self._pairwise_table, pair_rows) + + release_time_rows = [] + # For py2 and py3, the release time info are the same, so we only + # record the py3 version by default. And also record py2 only packages. + for cs in compatibility_statuses: + if len(cs.packages) == 1: + if cs.python_major_version == '3' or \ + cs.packages[0].install_name in configs.PY2_ONLY_PKGS: + row = self._compatibility_status_to_release_time_row(cs) + if row: + release_time_rows.extend(row) + + if release_time_rows: + self._client.insert_rows( + self._release_time_table, + release_time_rows) diff --git a/compatibility_lib/compatibility_lib/configs.py b/compatibility_lib/compatibility_lib/configs.py index 7b6be552..513043db 100644 --- a/compatibility_lib/compatibility_lib/configs.py +++ b/compatibility_lib/compatibility_lib/configs.py @@ -58,3 +58,7 @@ 'gcloud', 'oauth2client', ] + +PY2_ONLY_PKGS = [ + 'google-cloud-dataflow', +] From ecae3f5706dbfe6fa87f068b25eaa99941edca09 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 16 Jul 2018 11:25:23 -0700 Subject: [PATCH 2/3] Address comments --- .../compatibility_lib/compatibility_store.py | 25 ++++++++++--------- .../compatibility_lib/configs.py | 4 --- .../get_compatibility_data.py | 5 +++- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/compatibility_lib/compatibility_lib/compatibility_store.py b/compatibility_lib/compatibility_lib/compatibility_store.py index 464ce8be..5d8345ef 100644 --- a/compatibility_lib/compatibility_lib/compatibility_store.py +++ b/compatibility_lib/compatibility_lib/compatibility_store.py @@ -22,7 +22,6 @@ from google.cloud import bigquery from google.cloud.bigquery import table -from compatibility_lib import configs from compatibility_lib import package _DATASET_NAME = 'compatibility_checker' @@ -48,6 +47,8 @@ class CompatibilityResult: status: The overall result of the compatibility check. details: A text description of the compatibility check. Will be None if the check succeeded. + dependency_info: The dict contains the dependency version info and + release time info. timestamp: The time at which the compatibility check was performed. """ @@ -178,7 +179,7 @@ def _compatibility_status_to_row( @staticmethod def _compatibility_status_to_release_time_row( - cs: CompatibilityResult) -> List: + cs: CompatibilityResult) -> List[Mapping[str, Any]]: """Converts a CompatibilityResult into a dict which is a row for release time table.""" if len(cs.packages) != 1 or cs.dependency_info is None: @@ -380,6 +381,7 @@ def save_compatibility_statuses( if any(cs for cs in compatibility_statuses if len(cs.packages) not in [1, 2]): raise ValueError('CompatibilityResult must have 1 or 2 packages') + rows = [self._compatibility_status_to_row(s) for s in compatibility_statuses] @@ -394,18 +396,17 @@ def save_compatibility_statuses( self._pairwise_table, pair_rows) - release_time_rows = [] - # For py2 and py3, the release time info are the same, so we only - # record the py3 version by default. And also record py2 only packages. + release_time_rows = {} for cs in compatibility_statuses: if len(cs.packages) == 1: - if cs.python_major_version == '3' or \ - cs.packages[0].install_name in configs.PY2_ONLY_PKGS: - row = self._compatibility_status_to_release_time_row(cs) - if row: - release_time_rows.extend(row) + install_name = cs.packages[0].install_name + row = self._compatibility_status_to_release_time_row(cs) + if row: + release_time_rows[install_name] = row + + print(release_time_rows) - if release_time_rows: + for row in release_time_rows.values(): self._client.insert_rows( self._release_time_table, - release_time_rows) + row) diff --git a/compatibility_lib/compatibility_lib/configs.py b/compatibility_lib/compatibility_lib/configs.py index 513043db..7b6be552 100644 --- a/compatibility_lib/compatibility_lib/configs.py +++ b/compatibility_lib/compatibility_lib/configs.py @@ -58,7 +58,3 @@ 'gcloud', 'oauth2client', ] - -PY2_ONLY_PKGS = [ - 'google-cloud-dataflow', -] diff --git a/compatibility_lib/compatibility_lib/get_compatibility_data.py b/compatibility_lib/compatibility_lib/get_compatibility_data.py index 8c850394..2f0527ed 100644 --- a/compatibility_lib/compatibility_lib/get_compatibility_data.py +++ b/compatibility_lib/compatibility_lib/get_compatibility_data.py @@ -54,10 +54,13 @@ def _result_dict_to_compatibility_result(results, python_version): def write_to_status_table(): # Write self compatibility status to BigQuery + self_res_list = [] for py_version in [PY2, PY3]: results = checker.get_self_compatibility(py_version) res_list = _result_dict_to_compatibility_result(results, py_version) - store.save_compatibility_statuses(res_list) + self_res_list.extend(res_list) + + store.save_compatibility_statuses(self_res_list) # Write pairwise compatibility status to BigQuery for py_version in [PY2, PY3]: From 27698686a5e45b3f20b8335fce028e5be3514385 Mon Sep 17 00:00:00 2001 From: Angela Li Date: Mon, 16 Jul 2018 11:26:44 -0700 Subject: [PATCH 3/3] Remove print statement --- compatibility_lib/compatibility_lib/compatibility_store.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/compatibility_lib/compatibility_lib/compatibility_store.py b/compatibility_lib/compatibility_lib/compatibility_store.py index 5d8345ef..1eef35f2 100644 --- a/compatibility_lib/compatibility_lib/compatibility_store.py +++ b/compatibility_lib/compatibility_lib/compatibility_store.py @@ -404,8 +404,6 @@ def save_compatibility_statuses( if row: release_time_rows[install_name] = row - print(release_time_rows) - for row in release_time_rows.values(): self._client.insert_rows( self._release_time_table,