Fix python 3 support for pull request generation. (#489)

Calling `.encode('utf-8')` on a python2 unicode object encodes it into a
utf-8 str but calling it on a python3 str encodes that str into a bytes
object.

We want python str objects in both cases to be serialized into the
distribution yaml files. To achieve that we filter the modified
distribution repo dict recursively for unicode keys and values
and encoding them to utf-8 strings.

This also fixes some issues in the GitHub library related to Python 3.

bloom/commands/release.py

@@ -423,10 +423,24 @@ def get_relative_distribution_file_path(distro):
 
 
 def generate_release_tag(distro):
-    return ('release/%s/{package}/{version}' % distro).encode('utf-8')
+    tag = ('release/%s/{package}/{version}' % distro)
+    if sys.version_info[0] < 3:
+        tag == tag.encode('utf-8')
+    return tag
 
 
 def generate_ros_distro_diff(track, repository, distro, override_release_repository_url):
+    def convert_unicode_dict_to_str(d):
+        for key, value in d.items():
+            if type(key) == unicode:
+                del d[key]
+                key = key.encode('utf-8')
+                if type(value) == unicode:
+                    value = value.encode('utf-8')
+                if type(value) == dict:
+                    convert_unicode_dict_to_str(value)
+                d[key] = value
+
     global _user_provided_release_url
     distribution_dict = get_distribution_file(distro).get_data()
     # Get packages
@@ -437,28 +451,27 @@ def generate_ros_distro_diff(track, repository, distro, override_release_reposit
     track_dict = get_tracks_dict_raw()['tracks'][track]
     last_version = track_dict['last_version']
     release_inc = track_dict['release_inc']
-    version = '{0}-{1}'.format(last_version, release_inc).encode('utf-8')
     # Create a repository if there isn't already one
     if repository not in distribution_dict['repositories']:
         distribution_dict['repositories'][repository] = {}
     # Create a release entry if there isn't already one
     if 'release' not in distribution_dict['repositories'][repository]:
-        distribution_dict['repositories'][repository]['release'.encode('utf-8')] = {
-            'url'.encode('utf-8'): override_release_repository_url or _user_provided_release_url
+        distribution_dict['repositories'][repository]['release'] = {
+            'url': override_release_repository_url or _user_provided_release_url
         }
     # Update the repository
     repo = distribution_dict['repositories'][repository]['release']
     # Consider the override
     if override_release_repository_url is not None:
-        repo['url'.encode('utf-8')] = override_release_repository_url
+        repo['url'] = override_release_repository_url
     if 'tags' not in repo:
-        repo['tags'.encode('utf-8')] = {}
-    repo['tags']['release'.encode('utf-8')] = generate_release_tag(distro)
-    repo['version'.encode('utf-8')] = version
+        repo['tags'] = {}
+    repo['tags']['release'] = generate_release_tag(distro)
+    repo['version'] = '{0}-{1}'.format(last_version, release_inc)
     if 'last_release' in track_dict:
-        repo['upstream_tag'.encode('utf-8')] = track_dict['last_release']
+        repo['upstream_tag'] = track_dict['last_release']
     if 'packages' not in repo:
-        repo['packages'.encode('utf-8')] = []
+        repo['packages'] = []
     for path, pkg in packages.items():
         if pkg.name not in repo['packages']:
             repo['packages'].append(pkg.name)
@@ -468,6 +481,8 @@ def generate_ros_distro_diff(track, repository, distro, override_release_reposit
         if pkg_name not in packages_being_released:
             repo['packages'].remove(pkg_name)
     repo['packages'].sort()
+    if sys.version_info[0] < 3:
+        convert_unicode_dict_to_str(repo)
 
     def get_repository_info_from_user(url_type, defaults=None):
         data = {}
@@ -604,7 +619,7 @@ def get_repository_info_from_user(url_type, defaults=None):
     distro_file_name = get_relative_distribution_file_path(distro)
     updated_distribution_file = rosdistro.DistributionFile(distro, distribution_dict)
     distro_dump = yaml_from_distribution_file(updated_distribution_file)
-    distro_file_raw = load_url_to_file_handle(get_distribution_file_url(distro)).read()
+    distro_file_raw = load_url_to_file_handle(get_distribution_file_url(distro)).read().decode('utf-8')
     if distro_file_raw != distro_dump:
         # Calculate the diff
         udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(),

bloom/github.py

@@ -41,6 +41,7 @@
 import datetime
 import json
 import socket
+import sys
 
 
 try:
@@ -87,7 +88,10 @@ def do_github_post_req(path, data=None, auth=None, site='api.github.com'):
     if data is None:
         request = Request(url, headers=headers)  # GET
     else:
-        request = Request(url, data=json.dumps(data), headers=headers)  # POST
+        data = json.dumps(data)
+        if sys.version_info[0] >= 3:
+            data = data.encode('utf-8')
+        request = Request(url, data=data, headers=headers)  # POST
 
     try:
         response = urlopen(request, timeout=120)