From 04730488bc3732ccd5093d3feb1ad1ce6a1dfdcf Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Mon, 13 Nov 2017 16:44:21 -0800 Subject: [PATCH 1/5] If parse_version just gives us a regular tuple use that for formatting --- .../apache_beam/runners/dataflow/internal/dependency.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py index fba2df275517..703cff15d2d6 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py @@ -547,8 +547,12 @@ def _get_required_container_version(job_type=None): version = pkg_resources.get_distribution(GOOGLE_PACKAGE_NAME).version # We drop any pre/post parts of the version and we keep only the X.Y.Z # format. For instance the 0.3.0rc2 SDK version translates into 0.3.0. - container_version = ( - '%s.%s.%s' % pkg_resources.parse_version(version)._version.release) + parsed_version = pkg_resources.parse_version(version) + if "_version" in parsed_version: + container_version = ( + '%s.%s.%s' % parsed_version._version.release) + else: + container_version = ('%s.%s.%s' % parsed_version) # We do, however, keep the ".dev" suffix if it is present. if re.match(r'.*\.dev[0-9]*$', version): container_version += '.dev' From 05956f83e2437748c270cd0da49383f31784a36d Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Mon, 13 Nov 2017 17:27:34 -0800 Subject: [PATCH 2/5] Just do try fall through to except --- .../apache_beam/runners/dataflow/internal/dependency.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py index 703cff15d2d6..a70d5b998981 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py @@ -548,10 +548,10 @@ def _get_required_container_version(job_type=None): # We drop any pre/post parts of the version and we keep only the X.Y.Z # format. For instance the 0.3.0rc2 SDK version translates into 0.3.0. parsed_version = pkg_resources.parse_version(version) - if "_version" in parsed_version: + try: container_version = ( '%s.%s.%s' % parsed_version._version.release) - else: + except: container_version = ('%s.%s.%s' % parsed_version) # We do, however, keep the ".dev" suffix if it is present. if re.match(r'.*\.dev[0-9]*$', version): From 1e0f6ff9b8cca73f78e381779e27dde510d11d75 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Mon, 13 Nov 2017 20:27:26 -0800 Subject: [PATCH 3/5] Check exception and add comment --- .../python/apache_beam/runners/dataflow/internal/dependency.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py index a70d5b998981..95434253317d 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py @@ -548,10 +548,11 @@ def _get_required_container_version(job_type=None): # We drop any pre/post parts of the version and we keep only the X.Y.Z # format. For instance the 0.3.0rc2 SDK version translates into 0.3.0. parsed_version = pkg_resources.parse_version(version) + # Fall through for different results from parse_version. try: container_version = ( '%s.%s.%s' % parsed_version._version.release) - except: + except AttributeError: container_version = ('%s.%s.%s' % parsed_version) # We do, however, keep the ".dev" suffix if it is present. if re.match(r'.*\.dev[0-9]*$', version): From 16b9b43fce3118ae6569f056450cc0f4e607cec7 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Thu, 30 Nov 2017 16:14:05 -0800 Subject: [PATCH 4/5] Don't parse the release version just directly use the get_distribution version result --- .../runners/dataflow/internal/dependency.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py index 95434253317d..ccbbec0cccd4 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py @@ -544,20 +544,7 @@ def _get_required_container_version(job_type=None): """ # TODO(silviuc): Handle apache-beam versions when we have official releases. try: - version = pkg_resources.get_distribution(GOOGLE_PACKAGE_NAME).version - # We drop any pre/post parts of the version and we keep only the X.Y.Z - # format. For instance the 0.3.0rc2 SDK version translates into 0.3.0. - parsed_version = pkg_resources.parse_version(version) - # Fall through for different results from parse_version. - try: - container_version = ( - '%s.%s.%s' % parsed_version._version.release) - except AttributeError: - container_version = ('%s.%s.%s' % parsed_version) - # We do, however, keep the ".dev" suffix if it is present. - if re.match(r'.*\.dev[0-9]*$', version): - container_version += '.dev' - return container_version + return pkg_resources.get_distribution(GOOGLE_PACKAGE_NAME).version except pkg_resources.DistributionNotFound: # This case covers Apache Beam end-to-end testing scenarios. All these tests # will run with a special container version. From ab8b9dceefccdcac690c269a7ee325ef4469f366 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Fri, 15 Dec 2017 10:18:36 +0100 Subject: [PATCH 5/5] Remove unused import --- sdks/python/apache_beam/runners/dataflow/internal/dependency.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py index ccbbec0cccd4..fc0ce2cfd6ef 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py @@ -54,7 +54,6 @@ import glob import logging import os -import re import shutil import sys import tempfile