Enable github private repos.

.gitignore

@@ -25,3 +25,4 @@ MANIFEST
 *.DS_Store
 deb_dist
 .fuse_*
+*.swp

bloom/commands/release.py

@@ -36,7 +36,6 @@
 
 import argparse
 import atexit
-import base64
 import datetime
 import difflib
 import getpass
@@ -73,8 +72,6 @@
 from bloom.git import inbranch
 from bloom.git import ls_tree
 
-from bloom.github import auth_header_from_basic_auth
-from bloom.github import auth_header_from_oauth_token
 from bloom.github import Github
 from bloom.github import GithubException
 
@@ -104,6 +101,7 @@
 from bloom.util import safe_input
 from bloom.util import temporary_directory
 from bloom.util import to_unicode
+from bloom.util import auth_header
 
 try:
     import vcstools
@@ -395,7 +393,7 @@ def get_relative_distribution_file_path(distro):
 
 
 def generate_release_tag(distro):
-    return ('release/%s/{package}/{version}' % distro).encode('utf-8')
+    return 'release/%s/{package}/{version}' % distro
 
 
 def generate_ros_distro_diff(track, repository, distro, override_release_repository_url):
@@ -409,26 +407,26 @@ def generate_ros_distro_diff(track, repository, distro, override_release_reposit
     track_dict = get_tracks_dict_raw()['tracks'][track]
     last_version = track_dict['last_version']
     release_inc = track_dict['release_inc']
-    version = '{0}-{1}'.format(last_version, release_inc).encode('utf-8')
+    version = '{0}-{1}'.format(last_version, release_inc)
     # Create a repository if there isn't already one
     if repository not in distribution_dict['repositories']:
         distribution_dict['repositories'][repository] = {}
     # Create a release entry if there isn't already one
     if 'release' not in distribution_dict['repositories'][repository]:
-        distribution_dict['repositories'][repository]['release'.encode('utf-8')] = {
-            'url'.encode('utf-8'): override_release_repository_url or _user_provided_release_url
+        distribution_dict['repositories'][repository]['release'] = {
+            'url': override_release_repository_url or _user_provided_release_url
         }
     # Update the repository
     repo = distribution_dict['repositories'][repository]['release']
     # Consider the override
     if override_release_repository_url is not None:
-        repo['url'.encode('utf-8')] = override_release_repository_url
+        repo['url'] = override_release_repository_url
     if 'tags' not in repo:
-        repo['tags'.encode('utf-8')] = {}
-    repo['tags']['release'.encode('utf-8')] = generate_release_tag(distro)
-    repo['version'.encode('utf-8')] = version
+        repo['tags'] = {}
+    repo['tags']['release'] = generate_release_tag(distro)
+    repo['version'] = version
     if 'packages' not in repo:
-        repo['packages'.encode('utf-8')] = []
+        repo['packages'] = []
     for path, pkg in packages.items():
         if pkg.name not in repo['packages']:
             repo['packages'].append(pkg.name)
@@ -577,8 +575,10 @@ def get_repository_info_from_user(url_type, defaults=None):
     distro_file_raw = load_url_to_file_handle(get_distribution_file_url(distro)).read()
     if distro_file_raw != distro_dump:
         # Calculate the diff
-        udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(),
-                                     fromfile=distro_file_name, tofile=distro_file_name)
+        udiff = difflib.unified_diff(
+                list(map(str, distro_file_raw.splitlines())),
+                list(map(str, distro_dump.splitlines())),
+                fromfile=distro_file_name, tofile=distro_file_name)
         temp_dir = tempfile.mkdtemp()
         udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch')
         udiff_raw = ''
@@ -625,7 +625,6 @@ def get_repository_info_from_user(url_type, defaults=None):
 
 
 def get_gh_info(url):
-    from urlparse import urlparse
     o = urlparse(url)
     if 'raw.github.com' not in o.netloc and 'raw.githubusercontent.com' not in o.netloc:
         return None, None, None, None
@@ -651,7 +650,7 @@ def get_github_interface(quiet=False):
             token = config.get('oauth_token', None)
             username = config.get('github_user', None)
             if token and username:
-                return Github(username, auth=auth_header_from_oauth_token(token), token=token)
+                return Github(username, auth=auth_header(token=token), token=token)
     if not os.path.isdir(os.path.dirname(oauth_config_path)):
         os.makedirs(os.path.dirname(oauth_config_path))
     if quiet:
@@ -679,7 +678,7 @@ def get_github_interface(quiet=False):
         if not password:
             error("No password was given, aborting.")
             return None
-        gh = Github(username, auth=auth_header_from_basic_auth(username, password))
+        gh = Github(username, auth=auth_header(username=username, password=password))
         try:
             token = gh.create_new_bloom_authorization(update_auth=True)
             with open(oauth_config_path, 'a') as f:

bloom/github.py

@@ -41,30 +41,24 @@
 import datetime
 import json
 import socket
+import os
 
-from urlparse import urlunsplit
-from urllib import urlencode
 
 try:
     # Python2
+    from urlparse import urlencode, urlunsplit
     from urllib2 import HTTPError
     from urllib2 import URLError
     from urllib2 import Request, urlopen
 except ImportError:
     # Python3
+    from urllib.parse import urlencode, urlunsplit
     from urllib.error import HTTPError
     from urllib.error import URLError
     from urllib.request import Request, urlopen
 
 import bloom
-
-
-def auth_header_from_basic_auth(user, password):
-    return "Basic {0}".format(base64.b64encode('{0}:{1}'.format(user, password)))
-
-
-def auth_header_from_oauth_token(token):
-    return "token " + token
+from bloom.util import auth_header, GITHUB_USER, GITHUB_PASSWORD
 
 
 def get_bloom_headers(auth=None):
@@ -86,8 +80,11 @@ def do_github_post_req(path, data=None, auth=None, site='api.github.com'):
     if data is None:
         request = Request(url, headers=headers)  # GET
     else:
-        request = Request(url, data=json.dumps(data), headers=headers)  # POST
+        request = Request(url, data=json.dumps(data).encode('utf-8'), headers=headers)  # POST
 
+    if GITHUB_USER and GITHUB_PASSWORD:
+        request.add_header('Authorization', auth_header(
+            username=GITHUB_USER, password=GITHUB_PASSWORD))
     try:
         response = urlopen(request, timeout=120)
     except HTTPError as e:
@@ -125,13 +122,13 @@ def create_new_bloom_authorization(self, note=None, note_url=None, scopes=None,
             "note_url": 'http://bloom.readthedocs.org/' if note_url is None else note_url
         }
         resp = do_github_post_req('/authorizations', payload, self.auth)
-        resp_data = json.loads(resp.read())
+        resp_data = json.loads(str(resp.read()))
         resp_code = '{0}'.format(resp.getcode())
         if resp_code not in ['201', '202'] or 'token' not in resp_data:
             raise GithubException("Failed to create a new oauth authorization", resp)
         token = resp_data['token']
         if update_auth:
-            self.auth = auth_header_from_oauth_token(token)
+            self.auth = auth_header(token=token)
             self.token = token
         return token
 
@@ -140,7 +137,7 @@ def get_repo(self, owner, repo):
         if '{0}'.format(resp.getcode()) not in ['200']:
             raise GithubException(
                 "Failed to get information for repository '{owner}/{repo}'".format(**locals()), resp)
-        resp_data = json.loads(resp.read())
+        resp_data = json.loads(str(resp.read()))
         return resp_data
 
     def list_repos(self, user, start_page=None):
@@ -152,7 +149,7 @@ def list_repos(self, user, start_page=None):
             if '{0}'.format(resp.getcode()) not in ['200']:
                 raise GithubException(
                     "Failed to list repositories for user '{user}' using url '{url}'".format(**locals()), resp)
-            new_repos = json.loads(resp.read())
+            new_repos = json.loads(str(resp.read()))
             if not new_repos:
                 return repos
             repos.extend(new_repos)
@@ -165,7 +162,8 @@ def get_branch(self, owner, repo, branch):
             raise GithubException("Failed to get branch information for '{branch}' on '{owner}/{repo}' using '{url}'"
                                   .format(**locals()),
                                   resp)
-        return json.loads(resp.read())
+        txt = resp.read().decode('utf-8')
+        return json.loads(txt)
 
     def list_branches(self, owner, repo, start_page=None):
         page = start_page or 1
@@ -176,7 +174,8 @@ def list_branches(self, owner, repo, start_page=None):
             if '{0}'.format(resp.getcode()) not in ['200']:
                 raise GithubException(
                     "Failed to list branches for '{owner}/{repo}' using url '{url}'".format(**locals()), resp)
-            new_branches = json.loads(resp.read())
+            txt = resp.read().decode('utf-8')
+            new_branches = json.loads(txt)
             if not new_branches:
                 return branches
             branches.extend(new_branches)
@@ -187,7 +186,8 @@ def create_fork(self, parent_org, parent_repo):
         if '{0}'.format(resp.getcode()) not in ['200', '202']:
             raise GithubException(
                 "Failed to create a fork of '{parent_org}/{parent_repo}'".format(**locals()), resp)
-        return json.loads(resp.read())
+        txt = resp.read().decode('utf-8')
+        return json.loads(txt)
 
     def list_forks(self, org, repo, start_page=None):
         page = start_page or 1
@@ -198,7 +198,8 @@ def list_forks(self, org, repo, start_page=None):
             if '{0}'.format(resp.getcode()) not in ['200', '202']:
                 raise GithubException(
                     "Failed to list forks of '{org}/{repo}'".format(**locals()), resp)
-            new_forks = json.loads(resp.read())
+            txt = resp.read().decode('utf-8')
+            new_forks = json.loads(txt)
             if not new_forks:
                 return forks
             forks.extend(new_forks)
@@ -214,5 +215,6 @@ def create_pull_request(self, org, repo, branch, fork_org, fork_branch, title, b
         resp = do_github_post_req('/repos/{org}/{repo}/pulls'.format(**locals()), data, self.auth)
         if '{0}'.format(resp.getcode()) not in ['200', '201']:
             raise GithubException("Failed to create pull request", resp)
-        resp_json = json.loads(resp.read())
+        txt = resp.read().decode('utf-8')
+        resp_json = json.loads(txt)
         return resp_json['html_url']

bloom/util.py

@@ -38,6 +38,7 @@
 
 import argparse
 import os
+import base64
 import shutil
 import socket
 import sys
@@ -48,12 +49,12 @@
     # Python2
     from urllib2 import HTTPError
     from urllib2 import URLError
-    from urllib2 import urlopen
+    from urllib2 import urlopen, Request
 except ImportError:
     # Python3
     from urllib.error import HTTPError
     from urllib.error import URLError
-    from urllib.request import urlopen
+    from urllib.request import urlopen, Request
 
 from email.utils import formatdate
 
@@ -78,6 +79,23 @@
 from bloom.logging import sanitize
 from bloom.logging import warning
 
+
+GITHUB_USER = os.getenv('GITHUB_USER', None)
+GITHUB_PASSWORD = os.getenv('GITHUB_PASSWORD', None)
+
+
+def auth_header(username=None, password=None, token=None):
+    if username and password:
+        if sys.version_info > (3, 0):
+            userandpass = base64.b64encode(bytes('%s:%s' % (username, password), 'utf-8'))
+        else:
+            userandpass = base64.b64encode('%s:%s' % (username, password))
+        userandpass = userandpass.decode('ascii')
+        return 'Basic %s' % userandpass
+    elif token:
+        return 'token %s' % token
+
+
 try:
     to_unicode = unicode
 except NameError:
@@ -195,8 +213,12 @@ def load_url_to_file_handle(url, retry=2, retry_period=1, timeout=10):
     :param timeout: timeout for opening the URL in seconds
     :type timeout: float
     """
+    req = Request(url)
+    if GITHUB_USER and GITHUB_PASSWORD:
+        req.add_header('Authorization', auth_header(
+            username=GITHUB_USER, password=GITHUB_PASSWORD))
     try:
-        fh = urlopen(url, timeout=timeout)
+        fh = urlopen(req, timeout=timeout)
     except HTTPError as e:
         if e.code == 503 and retry:
             time.sleep(retry_period)