ARROW-6963: [Packaging][Wheel][OSX] Use crossbow's command to deploy …

…artifacts from travis builds

Closes #5726 from kszucs/osx-wheel-upload and squashes the following commits:

320b76e <Krisztián Szűcs> install requests security for system python
4b42380 <Krisztián Szűcs> pass default paths as strings
80e2e5b <Krisztián Szűcs> better error if pygit2 is missing
1bc93a8 <Krisztián Szűcs> make pygit2 and github3 optional dependencies for crossbow

Authored-by: Krisztián Szűcs <szucs.krisztian@gmail.com>
Signed-off-by: Krisztián Szűcs <szucs.krisztian@gmail.com>

dev/tasks/crossbow.py

@@ -32,13 +32,21 @@
 
 import click
 import toolz
-import pygit2
-import github3
-import jira.client
-from jinja2 import Template, StrictUndefined
 from setuptools_scm.git import parse as parse_git_version
 from ruamel.yaml import YAML
 
+try:
+    import github3
+except ImportError:
+    github3 = object
+
+try:
+    import pygit2
+except ImportError:
+    PygitRemoteCallbacks = object
+else:
+    PygitRemoteCallbacks = pygit2.RemoteCallbacks
+
 
 CWD = Path(__file__).parent.absolute()
 
@@ -117,6 +125,7 @@ class JiraChangelog:
 
     def __init__(self, version, username, password,
                  server='https://issues.apache.org/jira'):
+        import jira.client
         self.server = server
         # clean version to the first numbers
         self.version = '.'.join(version.split('.')[:3])
@@ -201,7 +210,7 @@ def render(self, old_changelog, website=False):
         return out.getvalue().strip()
 
 
-class GitRemoteCallbacks(pygit2.RemoteCallbacks):
+class GitRemoteCallbacks(PygitRemoteCallbacks):
 
     def __init__(self, token):
         self.token = token
@@ -249,10 +258,10 @@ class Repo:
     def __init__(self, path, github_token=None, remote_url=None,
                  require_https=False):
         self.path = Path(path)
-        self.repo = pygit2.Repository(str(self.path))
         self.github_token = github_token
         self.require_https = require_https
         self._remote_url = remote_url
+        self._pygit_repo = None
         self._github_repo = None  # set by as_github_repo()
         self._updated_refs = []
 
@@ -267,6 +276,12 @@ def __str__(self):
             head=self.head
         )
 
+    @property
+    def repo(self):
+        if self._pygit_repo is None:
+            self._pygit_repo = pygit2.Repository(str(self.path))
+        return self._pygit_repo
+
     @property
     def origin(self):
         remote = self.repo.remotes['origin']
@@ -398,6 +413,10 @@ def file_contents(self, commit_id, file):
 
     def _parse_github_user_repo(self):
         m = re.match(r'.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', self.remote_url)
+        if m is None:
+            raise ValueError("Unable to parse the github owner and repository "
+                             "from the repository's remote url '{}'"
+                             .format(self.remote_url))
         user, repo = m.group(1), m.group(2)
         return user, repo
 
@@ -671,6 +690,7 @@ def __init__(self, platform, ci, template, artifacts=None, params=None):
         self._status = None  # status cache
 
     def render_files(self, **extra_params):
+        from jinja2 import Template, StrictUndefined
         path = CWD / self.template
         params = toolz.merge(self.params, extra_params)
         template = Template(path.read_text(), undefined=StrictUndefined)
@@ -1028,11 +1048,11 @@ def send(self, smtp_user, smtp_password, smtp_server, smtp_port):
 @click.option('--github-token', '-t', default=None,
               help='OAuth token for GitHub authentication')
 @click.option('--arrow-path', '-a',
-              type=click.Path(exists=True), default=DEFAULT_ARROW_PATH,
+              type=click.Path(exists=True), default=str(DEFAULT_ARROW_PATH),
               help='Arrow\'s repository path. Defaults to the repository of '
                    'this script')
 @click.option('--queue-path', '-q',
-              type=click.Path(exists=True), default=DEFAULT_QUEUE_PATH,
+              type=click.Path(exists=True), default=str(DEFAULT_QUEUE_PATH),
               help='The repository path used for scheduling the tasks. '
                    'Defaults to crossbow directory placed next to arrow')
 @click.option('--queue-remote', '-qr', default=None,
@@ -1059,7 +1079,7 @@ def crossbow(ctx, github_token, arrow_path, queue_path, queue_remote,
 
 @crossbow.command()
 @click.option('--changelog-path', '-c', type=click.Path(exists=True),
-              default=DEFAULT_ARROW_PATH / 'CHANGELOG.md',
+              default=str(DEFAULT_ARROW_PATH / 'CHANGELOG.md'),
               help='Path of changelog to update')
 @click.option('--arrow-version', '-v', default=None,
               help='Set target version explicitly')
@@ -1232,7 +1252,8 @@ def report(obj, job_name, sender_name, sender_email, recipient_email,
 
 @crossbow.command()
 @click.argument('job-name', required=True)
-@click.option('-t', '--target-dir', default=DEFAULT_ARROW_PATH / 'packages',
+@click.option('-t', '--target-dir',
+              default=str(DEFAULT_ARROW_PATH / 'packages'),
               type=click.Path(file_okay=False, dir_okay=True),
               help='Directory to download the build artifacts')
 @click.pass_obj

dev/tasks/python-wheels/travis.osx.yml

@@ -25,10 +25,10 @@ env:
   global:
     - PLAT=x86_64
     - TRAVIS_TAG={{ task.tag }}
-    - MACOSX_DEPLOYMENT_TARGET="10.9"
     - PYARROW_VERSION={{ arrow.no_rc_version }}
     - PYARROW_BUILD_VERBOSE=1
     - MB_PYTHON_VERSION={{ python_version }}
+    - MACOSX_DEPLOYMENT_TARGET="10.9"
 
 before_install:
   - git clone https://github.com/matthew-brett/multibuild # TODO pin it
@@ -40,29 +40,26 @@ before_install:
 
   # ARROW-3976 Old versions of git can cause failures when Homebrew prints a
   # donation solicitation. Attempt to update git
-  - git --version
-  - brew upgrade git
+  - brew upgrade git python
 
   # Also remove artifacts that depend on Boost
   - brew uninstall boost cgal postgis sfcgal
   - brew update
   - brew upgrade cmake
   - travis_wait 30 brew install bison flex grpc openssl@1.1 llvm@7 zlib gperftools
-  # Remove shared grpc libraries installed by brew to make sure
+  # remove shared grpc libraries installed by brew to make sure
   # we are linked against the static ones.
   - rm -f /usr/local/opt/grpc/lib/*.dylib
 
+  # source utilities required for wheel builds
   - export CONFIG_PATH=`pwd`/arrow/dev/tasks/python-wheels/osx-build.sh
   - source multibuild/common_utils.sh
   - source multibuild/travis_osx_steps.sh
-
   - before_install
   # Fix SSL TLS issue for Python 3.5 on macOS
   - pip install requests[security]
 
 install:
-  - mkdir -p dist
-
   # the following functions are defined in osx-build.sh
   - build_wheel arrow
 
@@ -78,19 +75,13 @@ install:
   # run the import tests
   - run_import_tests
 
-  # move built wheels to a top level directory
-  - mv -v arrow/python/dist/* dist/
-  # reinstall openssl because travis' deployment script depends on it
   - brew install openssl@1.1
-
-deploy:
-  provider: releases
-  api_key: $CROSSBOW_GITHUB_TOKEN
-  file_glob: true
-  file: dist/*.whl
-  skip_cleanup: true
-  on:
-    tags: true
+  # before_install activates a virtualenv but we need the system python3
+  - deactivate
+  # crossbow dependencies for deployment
+  - unset MACOSX_DEPLOYMENT_TARGET
+  - pip3 install click ruamel.yaml setuptools_scm github3.py toolz requests[security]
+  - python3 arrow/dev/tasks/crossbow.py --queue-path $(pwd) --queue-remote {{ queue.remote_url }} upload-artifacts --sha {{ task.branch }} --tag {{ task.tag }} --pattern "arrow/python/dist/*.whl"
 
 notifications:
   email: