diff --git a/compatibility_lib/compatibility_lib/compatibility_store.py b/compatibility_lib/compatibility_lib/compatibility_store.py index c13ffa79..85b11d62 100644 --- a/compatibility_lib/compatibility_lib/compatibility_store.py +++ b/compatibility_lib/compatibility_lib/compatibility_store.py @@ -179,7 +179,7 @@ def _compatibility_status_to_row( return row @staticmethod - def _compatibility_status_to_release_time_row( + def _compatibility_status_to_release_time_rows( cs: CompatibilityResult) -> List[Mapping[str, Any]]: """Converts a CompatibilityResult into a dict which is a row for release time table.""" @@ -403,43 +403,86 @@ def save_compatibility_statuses( self._pairwise_table, pair_rows) - release_time_rows = {} + # Dependencies are not stored per Python version. This is not + # theoretically sound but is probably good enough in practice. + # + # If there are multiple compatibility results for the same package, + # use the dependencies with the highest version for that package. + # For example, if the following CompatibilityResults were passed to + # `save_compatibility_statuses`: + # + # cr1 = CompatibilityResult( + # packages=[Package('package1')], + # dependency_info={'package1': {'installed_version': '1.2.3' ...}) + # cr2 = CompatibilityResult( + # packages=[Package('package1')], + # dependency_info={'package1': {'installed_version': '1.2.4' ...}) + # + # then the dependency information for `cr2` would be saved because it + # is the newest version ('1.2.4' vs '1.2.3'). If the versions are the + # same then choose one arbitrarily. + # + # This check is done to prevent an old versions of apache-beam, which + # was accidentally released for Python 3, from having it's dependencies + # stored. It will also make sure that the Python 3 version of package + # dependencies are stored when Python 2 releases stop happening. + install_name_to_compatibility_result = {} for cs in compatibility_statuses: if len(cs.packages) == 1: install_name = cs.packages[0].install_name - # Only store the dep info for latest version of the package - # being checked. e.g. pip install apache-beam will have - # different version installed in py2/3. - if not self._should_update_dep_info( - cs, release_time_rows.get(install_name)): - continue - row = self._compatibility_status_to_release_time_row(cs) - if row: - release_time_rows[install_name] = row - - for row in release_time_rows.values(): + if install_name not in install_name_to_compatibility_result: + install_name_to_compatibility_result[install_name] = cs + else: + old_version = self._get_package_version( + install_name_to_compatibility_result[install_name]) + new_version = self._get_package_version(cs) + # TODO: Do not compare versions lexicographically. + # Lexicographically, '10' < '9'. + if new_version > old_version: + install_name_to_compatibility_result[install_name] = cs + + dependency_rows = itertools.chain( + *[self._compatibility_status_to_release_time_rows(cs) + for cs in install_name_to_compatibility_result.values()]) + + # Insert the dependency rows in a stable order to make testing more + # convenient. + dependency_rows = sorted( + dependency_rows, + key=lambda row: (row['install_name'], row['dep_name'])) + + if dependency_rows: self._client.insert_rows( self._release_time_table, - row) + dependency_rows) - def _should_update_dep_info(self, cs, dep_info_stored): - """Return True if the stored version is behind latest version.""" - if dep_info_stored is None: - return True + def _get_package_version(self, result: CompatibilityResult) -> str: + """Returns the version of the single package in a CompatibilityResult. - install_name = cs.packages[0].install_name - install_name_sanitized = install_name.split('[')[0] - installed_version = cs.dependency_info[ - install_name_sanitized]['installed_version'] + Args: + result: The compatibility result. This result must contain exactly + one package. + + Returns: + A string containing the version of the single package found in the + CompatibilityResult `packages` attribute. For example: + + cr1 = CompatibilityResult( + packages=[Package('package1')], + dependency_info={'package1': {'installed_version': '1.2.3' ..}) + _get_package_version(cr1) => '1.2.3' + """ + if len(result.packages) != 1: + raise ValueError('multiple packages found in CompatibilityResult') - installed_version_stored = '0' - for row in dep_info_stored: - if row['install_name'] == install_name \ - and row['dep_name'] == install_name_sanitized: - installed_version_stored = row['installed_version'] - break + install_name = result.packages[0].install_name + install_name_sanitized = install_name.split('[')[0] - return True if installed_version > installed_version_stored else False + for pkg, version_info in result.dependency_info.items(): + if pkg == install_name_sanitized: + return version_info['installed_version'] + raise ValueError('missing version information for {}'.format( + install_name_sanitized)) @retrying.retry(stop_max_attempt_number=7, wait_fixed=2000) diff --git a/compatibility_lib/compatibility_lib/test_compatibility_store.py b/compatibility_lib/compatibility_lib/test_compatibility_store.py index d27ad4f3..41611121 100644 --- a/compatibility_lib/compatibility_lib/test_compatibility_store.py +++ b/compatibility_lib/compatibility_lib/test_compatibility_store.py @@ -473,6 +473,129 @@ def MockClient(project=None): mock_client.insert_rows.assert_called_with( store._release_time_table, [row_release_time]) + def test_save_compatibility_statuses_release_time_for_latest_many_packages( + self): + mock_client = mock.Mock() + timestamp = '2018-07-17 03:01:06.11693 UTC' + status = compatibility_store.Status.SUCCESS + apache_beam_py2 = mock.Mock( + packages=[package.Package('apache-beam[gcp]')], + python_major_version='2', + status=status, + details=None, + dependency_info={ + 'six': { + 'installed_version': '9.9.9', + 'installed_version_time': '2018-05-12T16:26:31', + 'latest_version': '2.7.0', + 'current_time': '2018-07-13T17:11:29.140608', + 'latest_version_time': '2018-05-12T16:26:31', + 'is_latest': False, + } , + 'apache-beam': { + 'installed_version': '2.7.0', + 'installed_version_time': '2018-05-12T16:26:31', + 'latest_version': '2.7.0', + 'current_time': '2018-07-13T17:11:29.140608', + 'latest_version_time': '2018-05-12T16:26:31', + 'is_latest': True, + }}, + timestamp=timestamp) + apache_beam_py3 = mock.Mock( + packages=[package.Package('apache-beam[gcp]')], + python_major_version='3', + status=status, + details=None, + dependency_info={'apache-beam': { + 'installed_version': '2.2.0', + 'installed_version_time': '2018-05-12T16:26:31', + 'latest_version': '2.7.0', + 'current_time': '2018-07-13T17:11:29.140608', + 'latest_version_time': '2018-05-12T16:26:31', + 'is_latest': False, + }}, + timestamp=timestamp) + google_api_core_py2 = mock.Mock( + packages=[package.Package('google-api-core')], + python_major_version='2', + status=status, + details=None, + dependency_info={ + 'google-api-core': { + 'installed_version': '3.7.0', + 'installed_version_time': '2018-05-12T16:26:31', + 'latest_version': '2.7.0', + 'current_time': '2018-07-13T17:11:29.140608', + 'latest_version_time': '2018-05-12T16:26:31', + 'is_latest': True, + }}, + timestamp=timestamp) + google_api_core_py3 = mock.Mock( + packages=[package.Package('google-api-core')], + python_major_version='3', + status=status, + details=None, + dependency_info={'google-api-core': { + 'installed_version': '3.7.1', + 'installed_version_time': '2018-05-12T16:26:31', + 'latest_version': '2.7.0', + 'current_time': '2018-07-13T17:11:29.140608', + 'latest_version_time': '2018-05-12T16:26:31', + 'is_latest': False, + }}, + timestamp=timestamp) + + apache_beam_row = { + 'install_name': 'apache-beam[gcp]', + 'dep_name': 'apache-beam', + 'installed_version': '2.7.0', + 'installed_version_time': '2018-05-12T16:26:31', + 'latest_version': '2.7.0', + 'latest_version_time': '2018-05-12T16:26:31', + 'is_latest': True, + 'timestamp': '2018-07-13T17:11:29.140608', + } + + six_row = { + 'install_name': 'apache-beam[gcp]', + 'dep_name': 'six', + 'installed_version': '9.9.9', + 'installed_version_time': '2018-05-12T16:26:31', + 'latest_version': '2.7.0', + 'latest_version_time': '2018-05-12T16:26:31', + 'is_latest': False, + 'timestamp': '2018-07-13T17:11:29.140608', + } + + google_api_core_row = { + 'install_name': 'google-api-core', + 'dep_name': 'google-api-core', + 'installed_version': '3.7.1', + 'installed_version_time': '2018-05-12T16:26:31', + 'latest_version': '2.7.0', + 'latest_version_time': '2018-05-12T16:26:31', + 'is_latest': False, + 'timestamp': '2018-07-13T17:11:29.140608', + } + + def MockClient(project=None): + return mock_client + + patch_client = mock.patch( + 'compatibility_lib.compatibility_store.bigquery.Client', + MockClient) + + with patch_client: + store = compatibility_store.CompatibilityStore() + store.save_compatibility_statuses( + [apache_beam_py2, + apache_beam_py3, + google_api_core_py2, + google_api_core_py3]) + + mock_client.insert_rows.assert_called_with( + store._release_time_table, + [apache_beam_row, six_row, google_api_core_row]) class MockClient(object):