Skip to content
This repository was archived by the owner on Oct 21, 2022. It is now read-only.

Update dependencies using a single insert_rows. #227

Merged
brianquinlan merged 2 commits into from
Jan 23, 2019
Merged

Update dependencies using a single insert_rows. #227

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6609b80
Update dependencies using a single insert_rows.
brianquinlan Jan 23, 2019
2105206
Add a note to fix lexicographical version comparison.
brianquinlan Jan 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 72 additions & 29 deletions compatibility_lib/compatibility_lib/compatibility_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ def _compatibility_status_to_row(
return row

@staticmethod
def _compatibility_status_to_release_time_row(
def _compatibility_status_to_release_time_rows(
cs: CompatibilityResult) -> List[Mapping[str, Any]]:
"""Converts a CompatibilityResult into a dict which is a row for
release time table."""
Expand Down Expand Up @@ -403,43 +403,86 @@ def save_compatibility_statuses(
self._pairwise_table,
pair_rows)

release_time_rows = {}
# Dependencies are not stored per Python version. This is not
# theoretically sound but is probably good enough in practice.
#
# If there are multiple compatibility results for the same package,
# use the dependencies with the highest version for that package.
# For example, if the following CompatibilityResults were passed to
# `save_compatibility_statuses`:
#
# cr1 = CompatibilityResult(
# packages=[Package('package1')],
# dependency_info={'package1': {'installed_version': '1.2.3' ...})
# cr2 = CompatibilityResult(
# packages=[Package('package1')],
# dependency_info={'package1': {'installed_version': '1.2.4' ...})
#
# then the dependency information for `cr2` would be saved because it
# is the newest version ('1.2.4' vs '1.2.3'). If the versions are the
# same then choose one arbitrarily.
#
# This check is done to prevent an old versions of apache-beam, which
# was accidentally released for Python 3, from having it's dependencies
# stored. It will also make sure that the Python 3 version of package
# dependencies are stored when Python 2 releases stop happening.
Comment thread
liyanhui1228 marked this conversation as resolved.
install_name_to_compatibility_result = {}
for cs in compatibility_statuses:
if len(cs.packages) == 1:
install_name = cs.packages[0].install_name
# Only store the dep info for latest version of the package
# being checked. e.g. pip install apache-beam will have
# different version installed in py2/3.
if not self._should_update_dep_info(
cs, release_time_rows.get(install_name)):
continue
row = self._compatibility_status_to_release_time_row(cs)
if row:
release_time_rows[install_name] = row

for row in release_time_rows.values():
if install_name not in install_name_to_compatibility_result:
install_name_to_compatibility_result[install_name] = cs
else:
old_version = self._get_package_version(
install_name_to_compatibility_result[install_name])
new_version = self._get_package_version(cs)
# TODO: Do not compare versions lexicographically.
Copy link
Copy Markdown
Member

@liyanhui1228 liyanhui1228 Jan 23, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably use distutils.version to compare the version number?

>>>from distutils.version import StrictVersion
>>> StrictVersion("2.3.1") < StrictVersion("10.1.2")
True

Copy link
Copy Markdown
Contributor Author

@brianquinlan brianquinlan Jan 23, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep. I'll fix this in a follow-up PR.

# Lexicographically, '10' < '9'.
if new_version > old_version:
install_name_to_compatibility_result[install_name] = cs

dependency_rows = itertools.chain(
*[self._compatibility_status_to_release_time_rows(cs)
for cs in install_name_to_compatibility_result.values()])

# Insert the dependency rows in a stable order to make testing more
# convenient.
dependency_rows = sorted(
dependency_rows,
key=lambda row: (row['install_name'], row['dep_name']))

if dependency_rows:
self._client.insert_rows(
self._release_time_table,
row)
dependency_rows)

def _should_update_dep_info(self, cs, dep_info_stored):
"""Return True if the stored version is behind latest version."""
if dep_info_stored is None:
return True
def _get_package_version(self, result: CompatibilityResult) -> str:
"""Returns the version of the single package in a CompatibilityResult.

install_name = cs.packages[0].install_name
install_name_sanitized = install_name.split('[')[0]
installed_version = cs.dependency_info[
install_name_sanitized]['installed_version']
Args:
result: The compatibility result. This result must contain exactly
one package.

Returns:
A string containing the version of the single package found in the
CompatibilityResult `packages` attribute. For example:

cr1 = CompatibilityResult(
packages=[Package('package1')],
dependency_info={'package1': {'installed_version': '1.2.3' ..})
_get_package_version(cr1) => '1.2.3'
"""
if len(result.packages) != 1:
raise ValueError('multiple packages found in CompatibilityResult')

installed_version_stored = '0'
for row in dep_info_stored:
if row['install_name'] == install_name \
and row['dep_name'] == install_name_sanitized:
installed_version_stored = row['installed_version']
break
install_name = result.packages[0].install_name
install_name_sanitized = install_name.split('[')[0]

return True if installed_version > installed_version_stored else False
for pkg, version_info in result.dependency_info.items():
if pkg == install_name_sanitized:
return version_info['installed_version']
raise ValueError('missing version information for {}'.format(
install_name_sanitized))

@retrying.retry(stop_max_attempt_number=7,
wait_fixed=2000)
Expand Down
123 changes: 123 additions & 0 deletions compatibility_lib/compatibility_lib/test_compatibility_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,6 +473,129 @@ def MockClient(project=None):
mock_client.insert_rows.assert_called_with(
store._release_time_table, [row_release_time])

def test_save_compatibility_statuses_release_time_for_latest_many_packages(
self):
mock_client = mock.Mock()
timestamp = '2018-07-17 03:01:06.11693 UTC'
status = compatibility_store.Status.SUCCESS
apache_beam_py2 = mock.Mock(
packages=[package.Package('apache-beam[gcp]')],
python_major_version='2',
status=status,
details=None,
dependency_info={
'six': {
'installed_version': '9.9.9',
'installed_version_time': '2018-05-12T16:26:31',
'latest_version': '2.7.0',
'current_time': '2018-07-13T17:11:29.140608',
'latest_version_time': '2018-05-12T16:26:31',
'is_latest': False,
} ,
'apache-beam': {
'installed_version': '2.7.0',
'installed_version_time': '2018-05-12T16:26:31',
'latest_version': '2.7.0',
'current_time': '2018-07-13T17:11:29.140608',
'latest_version_time': '2018-05-12T16:26:31',
'is_latest': True,
}},
timestamp=timestamp)
apache_beam_py3 = mock.Mock(
packages=[package.Package('apache-beam[gcp]')],
python_major_version='3',
status=status,
details=None,
dependency_info={'apache-beam': {
'installed_version': '2.2.0',
'installed_version_time': '2018-05-12T16:26:31',
'latest_version': '2.7.0',
'current_time': '2018-07-13T17:11:29.140608',
'latest_version_time': '2018-05-12T16:26:31',
'is_latest': False,
}},
timestamp=timestamp)
google_api_core_py2 = mock.Mock(
packages=[package.Package('google-api-core')],
python_major_version='2',
status=status,
details=None,
dependency_info={
'google-api-core': {
'installed_version': '3.7.0',
'installed_version_time': '2018-05-12T16:26:31',
'latest_version': '2.7.0',
'current_time': '2018-07-13T17:11:29.140608',
'latest_version_time': '2018-05-12T16:26:31',
'is_latest': True,
}},
timestamp=timestamp)
google_api_core_py3 = mock.Mock(
packages=[package.Package('google-api-core')],
python_major_version='3',
status=status,
details=None,
dependency_info={'google-api-core': {
'installed_version': '3.7.1',
'installed_version_time': '2018-05-12T16:26:31',
'latest_version': '2.7.0',
'current_time': '2018-07-13T17:11:29.140608',
'latest_version_time': '2018-05-12T16:26:31',
'is_latest': False,
}},
timestamp=timestamp)

apache_beam_row = {
'install_name': 'apache-beam[gcp]',
'dep_name': 'apache-beam',
'installed_version': '2.7.0',
'installed_version_time': '2018-05-12T16:26:31',
'latest_version': '2.7.0',
'latest_version_time': '2018-05-12T16:26:31',
'is_latest': True,
'timestamp': '2018-07-13T17:11:29.140608',
}

six_row = {
'install_name': 'apache-beam[gcp]',
'dep_name': 'six',
'installed_version': '9.9.9',
'installed_version_time': '2018-05-12T16:26:31',
'latest_version': '2.7.0',
'latest_version_time': '2018-05-12T16:26:31',
'is_latest': False,
'timestamp': '2018-07-13T17:11:29.140608',
}

google_api_core_row = {
'install_name': 'google-api-core',
'dep_name': 'google-api-core',
'installed_version': '3.7.1',
'installed_version_time': '2018-05-12T16:26:31',
'latest_version': '2.7.0',
'latest_version_time': '2018-05-12T16:26:31',
'is_latest': False,
'timestamp': '2018-07-13T17:11:29.140608',
}

def MockClient(project=None):
return mock_client

patch_client = mock.patch(
'compatibility_lib.compatibility_store.bigquery.Client',
MockClient)

with patch_client:
store = compatibility_store.CompatibilityStore()
store.save_compatibility_statuses(
[apache_beam_py2,
apache_beam_py3,
google_api_core_py2,
google_api_core_py3])

mock_client.insert_rows.assert_called_with(
store._release_time_table,
[apache_beam_row, six_row, google_api_core_row])

class MockClient(object):

Expand Down