ARROW-8165: [Packaging] Make nightly wheels available on a PyPI server

Uploaded wheels are available at https://pypi.fury.io/ursa-labs/

Considered Bintray, but it doesn't provide any facilities for listing the wheels, although Artifactory could be a good solution, but it only has paid plans hosted on-prem at AWS.

Gemfury seems like a good solution, pros:
- it supports multiple repositories which can be relevant for us (nuget, npm, deb, rpm, gem)
- seems to be free and unlimited for public packages
- no client needed, simple curl command is enough to upload the binaries

Closes #6669 from kszucs/gemfury

Authored-by: Krisztián Szűcs <szucs.krisztian@gmail.com>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>

dev/tasks/conda-recipes/azure.linux.yml

@@ -55,7 +55,9 @@ jobs:
       CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN)
     displayName: Upload packages as a GitHub release
 
+  {% if arrow.branch == 'master' %}
   # Upload to custom anaconda channel
   - script: |
       anaconda -t $(CROSSBOW_ANACONDA_TOKEN) upload -l nightly --force build_artifacts/linux-64/*.tar.bz2
     displayName: Upload packages to Anaconda
+  {% endif %}

dev/tasks/conda-recipes/azure.osx.yml

@@ -87,10 +87,12 @@ jobs:
       CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN)
     displayName: Upload packages as a GitHub release
 
+  {% if arrow.branch == 'master' %}
   # Upload to custom anaconda channel
   - script: |
       source activate base
       conda install -y anaconda-client
       anaconda -t $(CROSSBOW_ANACONDA_TOKEN) upload -l nightly --force build_artifacts/osx-64/*.tar.bz2
     displayName: Upload packages to Anaconda
     workingDirectory: arrow/dev/tasks/conda-recipes
+  {% endif %}

dev/tasks/conda-recipes/azure.win.yml

@@ -102,10 +102,12 @@ jobs:
         CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN)
       displayName: Upload packages as a GitHub release
 
+    {% if arrow.branch == 'master' %}
     # Upload to custom anaconda channel
     - script: |
         source activate base
         conda install -y anaconda-client
         anaconda -t $(CROSSBOW_ANACONDA_TOKEN) upload -l nightly --force D:\bld\win-64\*.tar.bz2
       displayName: Upload packages to Anaconda
       workingDirectory: arrow/dev/tasks/conda-recipes
+    {% endif %}

dev/tasks/crossbow.py

@@ -602,6 +602,7 @@ def github_upload_asset_curl(self, release, path, name, mime):
 
         command = [
             'curl',
+            '--fail',
             '-H', "Authorization: token {}".format(self.github_token),
             '-H', "Content-Type: {}".format(mime),
             '--data-binary', '@{}'.format(path),

dev/tasks/python-wheels/appveyor.yml

@@ -53,6 +53,12 @@ build_script:
 after_build:
   # the artifacts must be uploaded from a directory relative to the build root
   - xcopy %ARROW_SRC%\python\dist\* wheels\
+{% if arrow.branch == 'master' %}
+  # upload to gemfury
+  - conda install -y curl
+  - for /f %%i in ('dir /b wheels\*.whl') do set WHEEL_PATH=wheels\%%i
+  - curl.exe -f -F "package=@%WHEEL_PATH%" "https://%CROSSBOW_GEMFURY_TOKEN%@push.fury.io/ursa-labs/"
+{% endif %}
 
 #on_finish:
   #- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))

dev/tasks/python-wheels/azure.linux.yml

@@ -90,3 +90,13 @@ jobs:
       env:
         CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN)
       displayName: Upload packages as a GitHub release
+
+    {% if arrow.branch == 'master' %}
+    # upload to gemfury
+    - script: |
+        path=$(ls arrow/python/{{ wheel_dir }}/dist/*.whl)
+        curl -f -F "package=@${path}" https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/ursa-labs/
+      env:
+        CROSSBOW_GEMFURY_TOKEN: $(CROSSBOW_GEMFURY_TOKEN)
+      displayName: Upload packages to Gemfury
+    {% endif %}

dev/tasks/python-wheels/travis.osx.yml

@@ -75,10 +75,23 @@ install:
   - sudo mv $(brew --cellar){.bak,}
   # before_install activates a virtualenv but we need the system python3
   - deactivate
+
   # crossbow dependencies for deployment
   - unset MACOSX_DEPLOYMENT_TARGET
   - pip3 install click ruamel.yaml setuptools_scm github3.py toolz requests[security]
-  - python3 arrow/dev/tasks/crossbow.py --queue-path $(pwd) --queue-remote {{ queue.remote_url }} upload-artifacts --sha {{ task.branch }} --tag {{ task.tag }} --pattern "arrow/python/dist/*.whl"
+  - python3 arrow/dev/tasks/crossbow.py
+      --queue-path $(pwd)
+      --queue-remote {{ queue.remote_url }}
+      upload-artifacts
+      --sha {{ task.branch }}
+      --tag {{ task.tag }}
+      --pattern "arrow/python/dist/*.whl"
+
+  {% if arrow.branch == 'master' %}
+  # upload to gemfury pypi repository, there should be a single wheel
+  - path=$(ls arrow/python/dist/*.whl)
+  - curl -f -F "package=@${path}" https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/ursa-labs/
+  {% endif %}
 
 notifications:
   email: