From faa183cc61dd8a6a2553e4c13540638018a43d40 Mon Sep 17 00:00:00 2001 From: jamarshon Date: Wed, 31 Jul 2019 09:57:05 -0400 Subject: [PATCH 01/17] Make test scripts runnable without being modules. (#187) --- test/{ => compliance}/__init__.py | 0 test/test.py | 4 ++-- .../test_kaldi.py => test_compliance_kaldi.py} | 10 +++++----- test/test_dataloader.py | 4 ++-- test/{datasets/test_vctk.py => test_datasets_vctk.py} | 4 ++-- test/test_functional.py | 4 ++-- test/test_kaldi_io.py | 4 ++-- test/test_legacy.py | 4 ++-- test/test_sox_effects.py | 4 ++-- test/test_transforms.py | 4 ++-- 10 files changed, 21 insertions(+), 21 deletions(-) rename test/{ => compliance}/__init__.py (100%) rename test/{compliance/test_kaldi.py => test_compliance_kaldi.py} (97%) rename test/{datasets/test_vctk.py => test_datasets_vctk.py} (94%) diff --git a/test/__init__.py b/test/compliance/__init__.py similarity index 100% rename from test/__init__.py rename to test/compliance/__init__.py diff --git a/test/test.py b/test/test.py index 0348a1c3a5..a26c379905 100644 --- a/test/test.py +++ b/test/test.py @@ -1,5 +1,5 @@ import unittest -import test.common_utils +import common_utils import torch import torchaudio import math @@ -7,7 +7,7 @@ class Test_LoadSave(unittest.TestCase): - test_dirpath, test_dir = test.common_utils.create_temp_assets_dir() + test_dirpath, test_dir = common_utils.create_temp_assets_dir() test_filepath = os.path.join(test_dirpath, "assets", "steam-train-whistle-daniel_simon.mp3") diff --git a/test/compliance/test_kaldi.py b/test/test_compliance_kaldi.py similarity index 97% rename from test/compliance/test_kaldi.py rename to test/test_compliance_kaldi.py index 0cd2e37ed5..9e235212df 100644 --- a/test/compliance/test_kaldi.py +++ b/test/test_compliance_kaldi.py @@ -1,7 +1,7 @@ import math import os -import test.common_utils -import test.compliance.utils +import common_utils +import compliance.utils import torch import torchaudio import torchaudio.compliance.kaldi as kaldi @@ -45,11 +45,11 @@ def first_sample_of_frame(frame, window_size, window_shift, snip_edges): class Test_Kaldi(unittest.TestCase): - test_dirpath, test_dir = test.common_utils.create_temp_assets_dir() + test_dirpath, test_dir = common_utils.create_temp_assets_dir() test_filepath = os.path.join(test_dirpath, 'assets', 'kaldi_file.wav') test_8000_filepath = os.path.join(test_dirpath, 'assets', 'kaldi_file_8000.wav') kaldi_output_dir = os.path.join(test_dirpath, 'assets', 'kaldi') - test_filepaths = {prefix: [] for prefix in test.compliance.utils.TEST_PREFIX} + test_filepaths = {prefix: [] for prefix in compliance.utils.TEST_PREFIX} # separating test files by their types (e.g 'spec', 'fbank', etc.) for f in os.listdir(kaldi_output_dir): @@ -151,7 +151,7 @@ def _compliance_test_helper(self, sound_filepath, filepath_key, expected_num_fil args = f.split('-') args[-1] = os.path.splitext(args[-1])[0] assert len(args) == expected_num_args, 'invalid test kaldi file name' - args = [test.compliance.utils.parse(arg) for arg in args] + args = [compliance.utils.parse(arg) for arg in args] output = get_output_fn(sound, args) diff --git a/test/test_dataloader.py b/test/test_dataloader.py index 46e5858c7a..f0c2f33442 100644 --- a/test/test_dataloader.py +++ b/test/test_dataloader.py @@ -1,5 +1,5 @@ import unittest -import test.common_utils +import common_utils import torch import torch.nn as nn from torch.utils.data import Dataset, DataLoader @@ -10,7 +10,7 @@ class TORCHAUDIODS(Dataset): - test_dirpath, test_dir = test.common_utils.create_temp_assets_dir() + test_dirpath, test_dir = common_utils.create_temp_assets_dir() def __init__(self): self.asset_dirpath = os.path.join(self.test_dirpath, "assets") diff --git a/test/datasets/test_vctk.py b/test/test_datasets_vctk.py similarity index 94% rename from test/datasets/test_vctk.py rename to test/test_datasets_vctk.py index 64907ea3e8..c205701bb5 100644 --- a/test/datasets/test_vctk.py +++ b/test/test_datasets_vctk.py @@ -3,13 +3,13 @@ import torch import torchaudio import unittest -import test.common_utils +import common_utils import torchaudio.datasets.vctk as vctk class TestVCTK(unittest.TestCase): def setUp(self): - self.test_dirpath, self.test_dir = test.common_utils.create_temp_assets_dir() + self.test_dirpath, self.test_dir = common_utils.create_temp_assets_dir() def get_full_path(self, file): return os.path.join(self.test_dirpath, 'assets', file) diff --git a/test/test_functional.py b/test/test_functional.py index 02b01620ed..f5571b1162 100644 --- a/test/test_functional.py +++ b/test/test_functional.py @@ -5,7 +5,7 @@ import torchaudio.functional as F import pytest import unittest -import test.common_utils +import common_utils from torchaudio.common_utils import IMPORT_LIBROSA @@ -30,7 +30,7 @@ def _test_istft_is_inverse_of_stft(self, kwargs): # operation to check whether we can reconstruct signal for data_size in self.data_sizes: for i in range(self.number_of_trials): - sound = test.common_utils.random_float_tensor(i, data_size) + sound = common_utils.random_float_tensor(i, data_size) stft = torch.stft(sound, **kwargs) estimate = torchaudio.functional.istft(stft, length=sound.size(1), **kwargs) diff --git a/test/test_kaldi_io.py b/test/test_kaldi_io.py index 08aeff54f6..f4ff557e56 100644 --- a/test/test_kaldi_io.py +++ b/test/test_kaldi_io.py @@ -2,13 +2,13 @@ import torch import torchaudio.kaldi_io as kio import unittest -import test.common_utils +import common_utils class Test_KaldiIO(unittest.TestCase): data1 = [[1, 2, 3], [11, 12, 13], [21, 22, 23]] data2 = [[31, 32, 33], [41, 42, 43], [51, 52, 53]] - test_dirpath, test_dir = test.common_utils.create_temp_assets_dir() + test_dirpath, test_dir = common_utils.create_temp_assets_dir() def _test_helper(self, file_name, expected_data, fn, expected_dtype): """ Takes a file_name to the input data and a function fn to extract the diff --git a/test/test_legacy.py b/test/test_legacy.py index 83af035daa..28e0b62432 100644 --- a/test/test_legacy.py +++ b/test/test_legacy.py @@ -1,5 +1,5 @@ import unittest -import test.common_utils +import common_utils import torch import torchaudio from torchaudio.legacy import save, load @@ -8,7 +8,7 @@ class Test_LoadSave(unittest.TestCase): - test_dirpath, test_dir = test.common_utils.create_temp_assets_dir() + test_dirpath, test_dir = common_utils.create_temp_assets_dir() test_filepath = os.path.join(test_dirpath, "assets", "steam-train-whistle-daniel_simon.mp3") diff --git a/test/test_sox_effects.py b/test/test_sox_effects.py index 2b35a841be..6ebc6eda4b 100644 --- a/test/test_sox_effects.py +++ b/test/test_sox_effects.py @@ -1,5 +1,5 @@ import unittest -import test.common_utils +import common_utils import torch import torchaudio import math @@ -7,7 +7,7 @@ class Test_SoxEffectsChain(unittest.TestCase): - test_dirpath, test_dir = test.common_utils.create_temp_assets_dir() + test_dirpath, test_dir = common_utils.create_temp_assets_dir() test_filepath = os.path.join(test_dirpath, "assets", "steam-train-whistle-daniel_simon.mp3") diff --git a/test/test_transforms.py b/test/test_transforms.py index b15ba40b94..a5a02e9ba5 100644 --- a/test/test_transforms.py +++ b/test/test_transforms.py @@ -7,7 +7,7 @@ from torchaudio.common_utils import IMPORT_LIBROSA, IMPORT_SCIPY import torchaudio.transforms as transforms import unittest -import test.common_utils +import common_utils if IMPORT_LIBROSA: import librosa @@ -26,7 +26,7 @@ class Tester(unittest.TestCase): waveform.unsqueeze_(0) # (1, 64000) waveform = (waveform * volume * 2**31).long() # file for stereo stft test - test_dirpath, test_dir = test.common_utils.create_temp_assets_dir() + test_dirpath, test_dir = common_utils.create_temp_assets_dir() test_filepath = os.path.join(test_dirpath, 'assets', 'steam-train-whistle-daniel_simon.mp3') From 959af9ab7cd5d8f636efdc807c96a750e94aef17 Mon Sep 17 00:00:00 2001 From: jamarshon Date: Wed, 31 Jul 2019 13:57:12 -0400 Subject: [PATCH 02/17] Binary install instructions. (#192) --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index bd543cdd03..f5207a791c 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,29 @@ conda install -c conda-forge sox Installation ------------ +### Binaries + +To install pip wheels of 0.2.0, select the appropriate pip wheel for +your version of Python: + +``` +# Wheels for Python 2 are NOT supported + +# Python 3.5 +pip3 install http://download.pytorch.org/whl/torchaudio-0.2-cp35-cp35m-linux_x86_64.whl + +# Python 3.6 +pip3 install http://download.pytorch.org/whl/torchaudio-0.2-cp36-cp36m-linux_x86_64.whl + +# Python 3.7 +pip3 install http://download.pytorch.org/whl/torchaudio-0.2-cp37-cp37m-linux_x86_64.whl +``` + +### From Source + +If your system configuration is not among the supported configurations +above, you can build from source. + ```bash # Linux python setup.py install From dae6cb250ec39f9634de67cfaf1e1fe6f2455550 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 31 Jul 2019 11:22:20 -0700 Subject: [PATCH 03/17] Fix version string in setup.py Signed-off-by: Edward Z. Yang --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d2d78de7be..1a1e914a72 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ def check_env_flag(name, default=''): # Creating the version file cwd = os.path.dirname(os.path.abspath(__file__)) -version = '0.2.0a0' +version = '0.3.0a0' sha = 'Unknown' try: From 8fcd9fdc94f2ae01c61df8f17c1998e53efa1e55 Mon Sep 17 00:00:00 2001 From: jamarshon Date: Thu, 1 Aug 2019 10:10:23 -0400 Subject: [PATCH 04/17] Removal of torchaudio.legacy (#195) --- .../packaging/conda/torchaudio/meta.yaml | 1 - docs/source/index.rst | 1 - docs/source/legacy.rst | 19 -- test/test_legacy.py | 164 ------------------ torchaudio/__init__.py | 2 +- torchaudio/legacy.py | 57 ------ 6 files changed, 1 insertion(+), 243 deletions(-) delete mode 100644 docs/source/legacy.rst delete mode 100644 test/test_legacy.py delete mode 100644 torchaudio/legacy.py diff --git a/build_tools/packaging/conda/torchaudio/meta.yaml b/build_tools/packaging/conda/torchaudio/meta.yaml index 28505c9bf3..2e7d5d3944 100644 --- a/build_tools/packaging/conda/torchaudio/meta.yaml +++ b/build_tools/packaging/conda/torchaudio/meta.yaml @@ -34,7 +34,6 @@ test: - torchaudio - torchaudio.datasets - torchaudio.kaldi_io - - torchaudio.legacy - torchaudio.sox_effects - torchaudio.transforms diff --git a/docs/source/index.rst b/docs/source/index.rst index 3420643ff1..c6d0efde69 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -13,7 +13,6 @@ The :mod:`torchaudio` package consists of I/O, popular datasets and common audio kaldi_io transforms functional - legacy .. automodule:: torchaudio :members: diff --git a/docs/source/legacy.rst b/docs/source/legacy.rst deleted file mode 100644 index ac95a657f7..0000000000 --- a/docs/source/legacy.rst +++ /dev/null @@ -1,19 +0,0 @@ -.. role:: hidden - :class: hidden-section - -torchaudio.legacy -====================== - -.. currentmodule:: torchaudio.legacy - -Legacy loading and save functions. - -:hidden:`load` -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autofunction:: load - -:hidden:`save` -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autofunction:: save diff --git a/test/test_legacy.py b/test/test_legacy.py deleted file mode 100644 index 28e0b62432..0000000000 --- a/test/test_legacy.py +++ /dev/null @@ -1,164 +0,0 @@ -import unittest -import common_utils -import torch -import torchaudio -from torchaudio.legacy import save, load -import math -import os - - -class Test_LoadSave(unittest.TestCase): - test_dirpath, test_dir = common_utils.create_temp_assets_dir() - test_filepath = os.path.join(test_dirpath, "assets", - "steam-train-whistle-daniel_simon.mp3") - - def test_load(self): - # check normal loading - x, sr = load(self.test_filepath) - self.assertEqual(sr, 44100) - self.assertEqual(x.size(), (278756, 2)) - self.assertGreater(x.sum(), 0) - - # check normalizing - x, sr = load(self.test_filepath, normalization=True) - self.assertEqual(x.dtype, torch.float32) - self.assertTrue(x.min() >= -1.0) - self.assertTrue(x.max() <= 1.0) - - # check raising errors - with self.assertRaises(OSError): - load("file-does-not-exist.mp3") - - with self.assertRaises(OSError): - tdir = os.path.join( - os.path.dirname(self.test_dirpath), "torchaudio") - load(tdir) - - def test_save(self): - # load signal - x, sr = load(self.test_filepath) - - # check save - new_filepath = os.path.join(self.test_dirpath, "test.wav") - save(new_filepath, x, sr) - self.assertTrue(os.path.isfile(new_filepath)) - os.unlink(new_filepath) - - # check automatic normalization - x /= 1 << 31 - save(new_filepath, x, sr) - self.assertTrue(os.path.isfile(new_filepath)) - os.unlink(new_filepath) - - # test save 1d tensor - x = x[:, 0] # get mono signal - x.squeeze_() # remove channel dim - save(new_filepath, x, sr) - self.assertTrue(os.path.isfile(new_filepath)) - os.unlink(new_filepath) - - # don't allow invalid sizes as inputs - with self.assertRaises(ValueError): - x.unsqueeze_(0) # N x L not L x N - save(new_filepath, x, sr) - - with self.assertRaises(ValueError): - x.squeeze_() - x.unsqueeze_(1) - x.unsqueeze_(0) # 1 x L x 1 - save(new_filepath, x, sr) - - # automatically convert sr from floating point to int - x.squeeze_(0) - save(new_filepath, x, float(sr)) - self.assertTrue(os.path.isfile(new_filepath)) - os.unlink(new_filepath) - - # don't save to folders that don't exist - with self.assertRaises(OSError): - new_filepath = os.path.join(self.test_dirpath, "no-path", - "test.wav") - save(new_filepath, x, sr) - - # save created file - sinewave_filepath = os.path.join(self.test_dirpath, "assets", - "sinewave.wav") - sr = 16000 - freq = 440 - volume = 0.3 - - y = (torch.cos( - 2 * math.pi * torch.arange(0, 4 * sr).float() * freq / sr)) - y.unsqueeze_(1) - # y is between -1 and 1, so must scale - y = (y * volume * 2**31).long() - save(sinewave_filepath, y, sr) - self.assertTrue(os.path.isfile(sinewave_filepath)) - - # test precision - new_filepath = os.path.join(self.test_dirpath, "test.wav") - si, ei = torchaudio.info(sinewave_filepath) - save(new_filepath, y, sr, precision=16) - si16, ei16 = torchaudio.info(new_filepath) - self.assertEqual(si.precision, 32) - self.assertEqual(si16.precision, 16) - os.unlink(new_filepath) - - def test_load_and_save_is_identity(self): - input_path = os.path.join(self.test_dirpath, 'assets', 'sinewave.wav') - tensor, sample_rate = load(input_path) - output_path = os.path.join(self.test_dirpath, 'test.wav') - save(output_path, tensor, sample_rate, 32) - tensor2, sample_rate2 = load(output_path) - self.assertTrue(tensor.allclose(tensor2)) - self.assertEqual(sample_rate, sample_rate2) - os.unlink(output_path) - - def test_load_partial(self): - num_frames = 100 - offset = 200 - # load entire mono sinewave wav file, load a partial copy and then compare - input_sine_path = os.path.join(self.test_dirpath, 'assets', 'sinewave.wav') - x_sine_full, sr_sine = load(input_sine_path) - x_sine_part, _ = load(input_sine_path, num_frames=num_frames, offset=offset) - l1_error = x_sine_full[offset:(num_frames + offset)].sub(x_sine_part).abs().sum().item() - # test for the correct number of samples and that the correct portion was loaded - self.assertEqual(x_sine_part.size(0), num_frames) - self.assertEqual(l1_error, 0.) - - # create a two channel version of this wavefile - x_2ch_sine = x_sine_full.repeat(1, 2) - out_2ch_sine_path = os.path.join(self.test_dirpath, 'assets', '2ch_sinewave.wav') - save(out_2ch_sine_path, x_2ch_sine, sr_sine) - x_2ch_sine_load, _ = load(out_2ch_sine_path, num_frames=num_frames, offset=offset) - os.unlink(out_2ch_sine_path) - l1_error = x_2ch_sine_load.sub(x_2ch_sine[offset:(offset + num_frames)]).abs().sum().item() - self.assertEqual(l1_error, 0.) - - # test with two channel mp3 - x_2ch_full, sr_2ch = load(self.test_filepath, normalization=True) - x_2ch_part, _ = load(self.test_filepath, normalization=True, num_frames=num_frames, offset=offset) - l1_error = x_2ch_full[offset:(offset + num_frames)].sub(x_2ch_part).abs().sum().item() - self.assertEqual(x_2ch_part.size(0), num_frames) - self.assertEqual(l1_error, 0.) - - # check behavior if number of samples would exceed file length - offset_ns = 300 - x_ns, _ = load(input_sine_path, num_frames=100000, offset=offset_ns) - self.assertEqual(x_ns.size(0), x_sine_full.size(0) - offset_ns) - - # check when offset is beyond the end of the file - with self.assertRaises(RuntimeError): - load(input_sine_path, offset=100000) - - def test_z_get_info(self): - input_path = os.path.join(self.test_dirpath, 'assets', 'sinewave.wav') - channels, samples, rate, precision = (1, 64000, 16000, 32) - si, ei = torchaudio.info(input_path) - self.assertEqual(si.channels, channels) - self.assertEqual(si.length, samples) - self.assertEqual(si.rate, rate) - self.assertEqual(ei.bits_per_sample, precision) - -if __name__ == '__main__': - unittest.main() diff --git a/torchaudio/__init__.py b/torchaudio/__init__.py index 942ba3530c..af25ea7c5e 100644 --- a/torchaudio/__init__.py +++ b/torchaudio/__init__.py @@ -5,7 +5,7 @@ import _torch_sox from .version import __version__, git_version -from torchaudio import transforms, datasets, kaldi_io, sox_effects, legacy, compliance, _docs +from torchaudio import transforms, datasets, kaldi_io, sox_effects, compliance, _docs def check_input(src): diff --git a/torchaudio/legacy.py b/torchaudio/legacy.py deleted file mode 100644 index 3d477fac3c..0000000000 --- a/torchaudio/legacy.py +++ /dev/null @@ -1,57 +0,0 @@ -from __future__ import division, print_function -import os.path - -import torch -import _torch_sox - -import torchaudio - - -def load(filepath, out=None, normalization=None, num_frames=0, offset=0): - r"""Loads an audio file from disk into a Tensor. The default options have - changed as of torchaudio 0.2 and this function maintains option defaults - from version 0.1. - - Args: - filepath (str): Path to audio file - out (torch.Tensor, optional): An output Tensor to use instead of creating one. (Default: ``None``) - normalization (bool or number, optional): If boolean `True`, then output is divided by `1 << 31` - (assumes 16-bit depth audio, and normalizes to `[0, 1]`. If `number`, then output is divided by that - number. (Default: ``None``) - num_frames (int, optional): Number of frames to load. -1 to load everything after the - offset. (Default: ``0``) - offset (int, optional): Number of frames from the start of the file to begin data - loading. (Default: ``0``) - - Returns: - Tuple[torch.Tensor, int]: The output tensor is of size `[L x C]` where L is the number of audio frames, - C is the number of channels. The integer is sample-rate of the audio (as listed in the metadata of - the file) - - Example - >>> data, sample_rate = torchaudio.legacy.load('foo.mp3') - >>> print(data.size()) - torch.Size([278756, 2]) - >>> print(sample_rate) - 44100 - """ - return torchaudio.load(filepath, out, normalization, False, num_frames, offset) - - -def save(filepath, src, sample_rate, precision=32): - r"""Saves a Tensor with audio signal to disk as a standard format like mp3, wav, etc. - The default options have changed as of torchaudio 0.2 and this function maintains - option defaults from version 0.1. - - Args: - filepath (str): Path to audio file - src (torch.Tensor): An input 2D Tensor of shape `[L x C]` where L is - the number of audio frames, C is the number of channels - sample_rate (int): The sample-rate of the audio to be saved - precision (int, optional): The bit-precision of the audio to be saved. (Default: ``32``) - - Example - >>> data, sample_rate = torchaudio.legacy.load('foo.mp3') - >>> torchaudio.legacy.save('foo.wav', data, sample_rate) - """ - torchaudio.save(filepath, src, sample_rate, precision, False) From ca16902300ff912c6ce0cdcbbc7b9b678e99b0aa Mon Sep 17 00:00:00 2001 From: jamarshon Date: Thu, 1 Aug 2019 18:10:13 -0400 Subject: [PATCH 05/17] Add missing from __future__ import division (#204) --- test/test_jit.py | 1 + torchaudio/functional.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_jit.py b/test/test_jit.py index ecccba246c..b1f13292e6 100644 --- a/test/test_jit.py +++ b/test/test_jit.py @@ -1,3 +1,4 @@ +from __future__ import division import torch import torchaudio.functional as F import torchaudio.transforms as transforms diff --git a/torchaudio/functional.py b/torchaudio/functional.py index 9155c827d7..bc24c805e9 100644 --- a/torchaudio/functional.py +++ b/torchaudio/functional.py @@ -1,3 +1,4 @@ +from __future__ import division import math import torch @@ -61,8 +62,8 @@ def istft(stft_matrix, # type: Tensor could be useful. If ``length`` is ``None`` then padding will be aggressively removed (some loss of signal). - [1] D. W. Griffin and J. S. Lim, “Signal estimation from modified short-time Fourier transform,” - IEEE Trans. ASSP, vol.32, no.2, pp.236–243, Apr. 1984. + [1] D. W. Griffin and J. S. Lim, "Signal estimation from modified short-time Fourier transform," + IEEE Trans. ASSP, vol.32, no.2, pp.236-243, Apr. 1984. Args: stft_matrix (torch.Tensor): Output of stft where each row of a channel is a frequency and each From b9cac74f5be772f040423a8ac3af92fb276aeae6 Mon Sep 17 00:00:00 2001 From: jamarshon Date: Thu, 1 Aug 2019 22:09:50 -0400 Subject: [PATCH 06/17] Packaging improvements for nightlies (#198) (#205) * Packaging improvements for nightlies - Instructions for how to install stable and master binaries, without hardcoding wheel filenames - Turn on Python 2 build (numba is only needed for testing) - Make builds a little more hermetic - Prefer building against nightly CPU-only torch build - Don't install all of requirements.txt, it's not necessary - Updates for OS X too, including portability issues Signed-off-by: Edward Z. Yang (cherry picked from commit 45423323ec10a4de166e1f2ee6f27cef47a5275c) --- .../packaging/wheel/build_from_source.sh | 7 ++++++- build_tools/packaging/wheel/linux_manywheel.sh | 17 +++++++++++------ build_tools/packaging/wheel/osx_wheel.sh | 6 ++---- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/build_tools/packaging/wheel/build_from_source.sh b/build_tools/packaging/wheel/build_from_source.sh index 665d3cfece..31279ac4ed 100755 --- a/build_tools/packaging/wheel/build_from_source.sh +++ b/build_tools/packaging/wheel/build_from_source.sh @@ -28,16 +28,21 @@ popd # build mad, statically pushd libmad-0.15.1b +# See https://stackoverflow.com/a/12864879/23845 +sed -i.bak 's/-march=i486//' configure ./configure --disable-shared --enable-static --prefix="$PREFIX/audio/third_party/mad" CFLAGS=-fPIC CXXFLAGS=-fPIC \ --with-pic --disable-debug --disable-dependency-tracking make -s -j && make install popd # build sox, statically +# --without-png makes OS X build less hazardous; somehow the build +# finds png and enables it. We don't want it; we'd need to package +# it statically if we do. pushd sox-14.4.2 ./configure --disable-shared --enable-static --prefix="$PREFIX/audio/third_party/sox" \ LDFLAGS="-L$PREFIX/audio/third_party/lame/lib -L$PREFIX/audio/third_party/flac/lib -L$PREFIX/audio/third_party/mad/lib" \ CPPFLAGS="-I$PREFIX/audio/third_party/lame/include -I$PREFIX/audio/third_party/flac/include -I$PREFIX/audio/third_party/mad/include" \ - --with-lame --with-flac --with-mad --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking + --with-lame --with-flac --with-mad --without-png --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking make -s -j && make install popd diff --git a/build_tools/packaging/wheel/linux_manywheel.sh b/build_tools/packaging/wheel/linux_manywheel.sh index 66e7daf7f5..9540a1ff5c 100755 --- a/build_tools/packaging/wheel/linux_manywheel.sh +++ b/build_tools/packaging/wheel/linux_manywheel.sh @@ -5,7 +5,7 @@ if [ "$#" -ne 1 ]; then fi export CUVER="$1" # cu90 cu100 cpu -export TORCHAUDIO_BUILD_VERSION="0.2.0" +export TORCHAUDIO_BUILD_VERSION="0.3.0" export TORCHAUDIO_BUILD_NUMBER="1" export OUT_DIR="/remote/$CUVER" @@ -27,13 +27,18 @@ export PREFIX="/tmp" cd /tmp/audio for PYDIR in "${python_installations[@]}"; do - # wheels for numba does not work with python 2.7 - if [[ "$PYDIR" == "/opt/python/cp27-cp27m/" || "$PYDIR" == "/opt/python/cp27-cp27mu/" ]]; then - continue; - fi export PATH=$PYDIR/bin:$OLD_PATH pip install --upgrade pip - pip install -r requirements.txt + + # For true hermetic builds, you ought to be constructing the docker + # from scratch each time. But this makes things marginally safer if + # you aren't doing this. + pip uninstall -y torch || true + pip uninstall -y torch_nightly || true + + # NB: do not actually install requirements.txt; that is only needed for + # testing + pip install torch numpy future IS_WHEEL=1 python setup.py clean IS_WHEEL=1 python setup.py bdist_wheel mkdir -p $OUT_DIR diff --git a/build_tools/packaging/wheel/osx_wheel.sh b/build_tools/packaging/wheel/osx_wheel.sh index 7f4cc53a9f..b018bb7ffe 100755 --- a/build_tools/packaging/wheel/osx_wheel.sh +++ b/build_tools/packaging/wheel/osx_wheel.sh @@ -11,7 +11,7 @@ wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh & . ~/minconda_wheel_env_tmp/bin/activate -export TORCHAUDIO_BUILD_VERSION="0.2.0" +export TORCHAUDIO_BUILD_VERSION="0.3.0" export TORCHAUDIO_BUILD_NUMBER="1" export OUT_DIR=~/torchaudio_wheels @@ -40,9 +40,7 @@ do conda create -yn $env_name python="$desired_python" conda activate $env_name - # install torchaudio dependencies - pip install -r requirements.txt - + pip install torch numpy future IS_WHEEL=1 python setup.py clean IS_WHEEL=1 python setup.py bdist_wheel mkdir -p $OUT_DIR From bbfd0411a6e1d829214be51a32f04d607328fd18 Mon Sep 17 00:00:00 2001 From: jamarshon Date: Mon, 5 Aug 2019 11:08:17 -0400 Subject: [PATCH 07/17] Make torchaudio work on Python 2. (#210) https://github.com/pytorch/audio/pull/209 --- .travis.yml | 5 +++-- requirements.txt | 5 ++++- test/common_utils.py | 3 ++- test/compliance/generate_fbank_data.py | 1 + test/compliance/generate_test_stft_data.py | 1 + test/compliance/utils.py | 1 + test/test.py | 1 + test/test_compliance_kaldi.py | 1 + test/test_dataloader.py | 1 + test/test_datasets_vctk.py | 1 + test/test_functional.py | 8 ++++++-- test/test_jit.py | 2 +- test/test_kaldi_io.py | 1 + test/test_sox_effects.py | 1 + test/test_transforms.py | 2 +- torchaudio/__init__.py | 2 +- torchaudio/_docs.py | 1 + torchaudio/common_utils.py | 1 + torchaudio/compliance/kaldi.py | 6 ++++-- torchaudio/datasets/vctk.py | 2 +- torchaudio/datasets/yesno.py | 2 +- torchaudio/functional.py | 2 +- torchaudio/kaldi_io.py | 1 + torchaudio/sox_effects.py | 2 +- torchaudio/transforms.py | 2 +- 25 files changed, 39 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index ae4ded7570..d202003b5f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,11 @@ cache: matrix: fast_finish: true include: - # TODO add this back in when there is a pytorch 1.2 for python 3.5 - # - env: PYTHON_VERSION="3.5" + - env: PYTHON_VERSION="3.7" - env: PYTHON_VERSION="3.6" + # TODO add this back in when there is a pytorch 1.2 for python 3.5 - env: PYTHON_VERSION="3.5" RUN_FLAKE8="true" SKIP_TESTS="true" + - env: PYTHON_VERSION="2.7" addons: apt: diff --git a/requirements.txt b/requirements.txt index 0de9fc4602..605990a164 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,7 @@ librosa>=0.4.3 scipy # Unit tests with pytest -pytest \ No newline at end of file +pytest + +# Testing only Py3 compat +backports.tempfile diff --git a/test/common_utils.py b/test/common_utils.py index a82c9ee691..a79f413d2b 100644 --- a/test/common_utils.py +++ b/test/common_utils.py @@ -1,6 +1,7 @@ +from __future__ import absolute_import, division, print_function, unicode_literals import os from shutil import copytree -import tempfile +import backports.tempfile as tempfile import torch TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__)) diff --git a/test/compliance/generate_fbank_data.py b/test/compliance/generate_fbank_data.py index d3ad613bee..6932b99b1f 100644 --- a/test/compliance/generate_fbank_data.py +++ b/test/compliance/generate_fbank_data.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function, unicode_literals import argparse import logging import os diff --git a/test/compliance/generate_test_stft_data.py b/test/compliance/generate_test_stft_data.py index baa94d0de5..fee1950015 100644 --- a/test/compliance/generate_test_stft_data.py +++ b/test/compliance/generate_test_stft_data.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function, unicode_literals import argparse import logging import os diff --git a/test/compliance/utils.py b/test/compliance/utils.py index 6f7c7e8cf8..004441efe0 100644 --- a/test/compliance/utils.py +++ b/test/compliance/utils.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function, unicode_literals import random import torchaudio diff --git a/test/test.py b/test/test.py index a26c379905..1253ea2d07 100644 --- a/test/test.py +++ b/test/test.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function, unicode_literals import unittest import common_utils import torch diff --git a/test/test_compliance_kaldi.py b/test/test_compliance_kaldi.py index 9e235212df..3289d2e702 100644 --- a/test/test_compliance_kaldi.py +++ b/test/test_compliance_kaldi.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function, unicode_literals import math import os import common_utils diff --git a/test/test_dataloader.py b/test/test_dataloader.py index f0c2f33442..2ab40dac43 100644 --- a/test/test_dataloader.py +++ b/test/test_dataloader.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function, unicode_literals import unittest import common_utils import torch diff --git a/test/test_datasets_vctk.py b/test/test_datasets_vctk.py index c205701bb5..4d3477c338 100644 --- a/test/test_datasets_vctk.py +++ b/test/test_datasets_vctk.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function, unicode_literals import os import torch diff --git a/test/test_functional.py b/test/test_functional.py index f5571b1162..8f4f84942d 100644 --- a/test/test_functional.py +++ b/test/test_functional.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function, unicode_literals import math import torch @@ -195,15 +196,18 @@ def _num_stft_bins(signal_len, fft_len, hop_length, pad): return (signal_len + 2 * pad - fft_len + hop_length) // hop_length -@pytest.mark.parametrize('rate', [0.5, 1.01, 1.3]) @pytest.mark.parametrize('complex_specgrams', [ torch.randn(1, 2, 1025, 400, 2), torch.randn(1, 1025, 400, 2) ]) +@pytest.mark.parametrize('rate', [0.5, 1.01, 1.3]) @pytest.mark.parametrize('hop_length', [256]) -@unittest.skipIf(not IMPORT_LIBROSA, 'Librosa is not available') def test_phase_vocoder(complex_specgrams, rate, hop_length): + # Using a decorator here causes parametrize to fail on Python 2 + if not IMPORT_LIBROSA: + raise unittest.SkipTest('Librosa is not available') + # Due to cummulative sum, numerical error in using torch.float32 will # result in bottom right values of the stretched sectrogram to not # match with librosa. diff --git a/test/test_jit.py b/test/test_jit.py index b1f13292e6..ac34269280 100644 --- a/test/test_jit.py +++ b/test/test_jit.py @@ -1,4 +1,4 @@ -from __future__ import division +from __future__ import absolute_import, division, print_function, unicode_literals import torch import torchaudio.functional as F import torchaudio.transforms as transforms diff --git a/test/test_kaldi_io.py b/test/test_kaldi_io.py index f4ff557e56..69d7ed7527 100644 --- a/test/test_kaldi_io.py +++ b/test/test_kaldi_io.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function, unicode_literals import os import torch import torchaudio.kaldi_io as kio diff --git a/test/test_sox_effects.py b/test/test_sox_effects.py index 6ebc6eda4b..4da71d8b7a 100644 --- a/test/test_sox_effects.py +++ b/test/test_sox_effects.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function, unicode_literals import unittest import common_utils import torch diff --git a/test/test_transforms.py b/test/test_transforms.py index a5a02e9ba5..b17c372f77 100644 --- a/test/test_transforms.py +++ b/test/test_transforms.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import absolute_import, division, print_function, unicode_literals import math import os diff --git a/torchaudio/__init__.py b/torchaudio/__init__.py index af25ea7c5e..dd89e45d4f 100644 --- a/torchaudio/__init__.py +++ b/torchaudio/__init__.py @@ -1,4 +1,4 @@ -from __future__ import division, print_function +from __future__ import absolute_import, division, print_function, unicode_literals import os.path import torch diff --git a/torchaudio/_docs.py b/torchaudio/_docs.py index 2b2c3000f6..74cfdfab83 100644 --- a/torchaudio/_docs.py +++ b/torchaudio/_docs.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function, unicode_literals import torchaudio diff --git a/torchaudio/common_utils.py b/torchaudio/common_utils.py index 2321ad26a6..bf96535de4 100644 --- a/torchaudio/common_utils.py +++ b/torchaudio/common_utils.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function, unicode_literals import sys PY3 = sys.version_info > (3, 0) diff --git a/torchaudio/compliance/kaldi.py b/torchaudio/compliance/kaldi.py index d0591f4411..c44d72340d 100644 --- a/torchaudio/compliance/kaldi.py +++ b/torchaudio/compliance/kaldi.py @@ -1,4 +1,6 @@ +from __future__ import absolute_import, division, print_function, unicode_literals import math +import fractions import random import torch @@ -600,7 +602,7 @@ def _get_LR_indices_and_weights(orig_freq, new_freq, output_samples_in_unit, win def _lcm(a, b): - return abs(a * b) // math.gcd(a, b) + return abs(a * b) // fractions.gcd(a, b) def _get_num_LR_output_samples(input_num_samp, samp_rate_in, samp_rate_out): @@ -675,7 +677,7 @@ def resample_waveform(waveform, orig_freq, new_freq, lowpass_filter_width=6): assert lowpass_cutoff * 2 <= min_freq - base_freq = math.gcd(int(orig_freq), int(new_freq)) + base_freq = fractions.gcd(int(orig_freq), int(new_freq)) input_samples_in_unit = int(orig_freq) // base_freq output_samples_in_unit = int(new_freq) // base_freq diff --git a/torchaudio/datasets/vctk.py b/torchaudio/datasets/vctk.py index d66c988303..2bfa7124f2 100644 --- a/torchaudio/datasets/vctk.py +++ b/torchaudio/datasets/vctk.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import absolute_import, division, print_function, unicode_literals import torch.utils.data as data import os import os.path diff --git a/torchaudio/datasets/yesno.py b/torchaudio/datasets/yesno.py index 8d80b9e14a..09e7c8aabc 100644 --- a/torchaudio/datasets/yesno.py +++ b/torchaudio/datasets/yesno.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import absolute_import, division, print_function, unicode_literals import torch.utils.data as data import os import os.path diff --git a/torchaudio/functional.py b/torchaudio/functional.py index bc24c805e9..1704ba5f2d 100644 --- a/torchaudio/functional.py +++ b/torchaudio/functional.py @@ -1,4 +1,4 @@ -from __future__ import division +from __future__ import absolute_import, division, print_function, unicode_literals import math import torch diff --git a/torchaudio/kaldi_io.py b/torchaudio/kaldi_io.py index 50b101db7a..a0520e66d9 100644 --- a/torchaudio/kaldi_io.py +++ b/torchaudio/kaldi_io.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, division, print_function, unicode_literals # To use this file, the dependency (https://github.com/vesis84/kaldi-io-for-python) # needs to be installed. This is a light wrapper around kaldi_io that returns # torch.Tensors. diff --git a/torchaudio/sox_effects.py b/torchaudio/sox_effects.py index 2c709503eb..0c058ca370 100644 --- a/torchaudio/sox_effects.py +++ b/torchaudio/sox_effects.py @@ -1,4 +1,4 @@ -from __future__ import division, print_function +from __future__ import absolute_import, division, print_function, unicode_literals import torch import _torch_sox diff --git a/torchaudio/transforms.py b/torchaudio/transforms.py index cdd079dccf..7e1872be98 100644 --- a/torchaudio/transforms.py +++ b/torchaudio/transforms.py @@ -1,4 +1,4 @@ -from __future__ import division, print_function +from __future__ import absolute_import, division, print_function, unicode_literals from warnings import warn import math import torch From b2c73b6067827e45f0433aae57833a9d222db68c Mon Sep 17 00:00:00 2001 From: jamarshon Date: Mon, 5 Aug 2019 14:38:47 -0400 Subject: [PATCH 08/17] Conda nightly packaging updates and Add missing build script. (#212) --- build_tools/packaging/conda/build_audio.sh | 22 +++++++++---------- .../packaging/conda/torchaudio/build.sh | 7 ++++++ .../packaging/conda/torchaudio/meta.yaml | 18 ++++++--------- .../packaging/wheel/build_from_source.sh | 16 ++++++++------ .../packaging/wheel/linux_manywheel.sh | 2 +- build_tools/packaging/wheel/osx_wheel.sh | 2 +- setup.py | 10 ++++----- 7 files changed, 40 insertions(+), 37 deletions(-) create mode 100644 build_tools/packaging/conda/torchaudio/build.sh diff --git a/build_tools/packaging/conda/build_audio.sh b/build_tools/packaging/conda/build_audio.sh index 3d32e13641..98518200d7 100755 --- a/build_tools/packaging/conda/build_audio.sh +++ b/build_tools/packaging/conda/build_audio.sh @@ -10,38 +10,38 @@ retry () { $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) } -export TORCHAUDIO_BUILD_VERSION="0.2.0" +export TORCHAUDIO_BUILD_VERSION="0.3.0" export TORCHAUDIO_BUILD_NUMBER=1 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" audio_rootdir="$(pwd)/torchaudio-src" +if [[ "$BRANCH" == "" ]]; then + BRANCH=master +fi + if [[ ! -d "$audio_rootdir" ]]; then rm -rf "$audio_rootdir" - git clone "https://github.com/pytorch/audio" "$audio_rootdir" - pushd "$audio_rootdir" - git checkout v$TORCHAUDIO_BUILD_VERSION - popd + git clone "https://github.com/pytorch/audio" "$audio_rootdir" -b "$BRANCH" fi +export TORCHAUDIO_GITHUB_ROOT_DIR="$audio_rootdir" + cd "$SOURCE_DIR" ANACONDA_USER=pytorch conda config --set anaconda_upload no -# "$desired_cuda" == 'cpu' -export TORCHAUDIO_PACKAGE_SUFFIX="" export CONDA_CUDATOOLKIT_CONSTRAINT="" export CUDA_VERSION="None" -if [[ "$OSTYPE" != "darwin"* ]]; then - export TORCHAUDIO_PACKAGE_SUFFIX="-cpu" -else +if [[ "$OSTYPE" == "darwin"* ]]; then export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ fi time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 torchaudio -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchaudio +# Currently disabled, see https://github.com/pytorch/pytorch/issues/23738 +# time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchaudio time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 torchaudio time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 torchaudio diff --git a/build_tools/packaging/conda/torchaudio/build.sh b/build_tools/packaging/conda/torchaudio/build.sh new file mode 100644 index 0000000000..7e5917060f --- /dev/null +++ b/build_tools/packaging/conda/torchaudio/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -ex + +PREFIX="$(pwd)" +. build_tools/packaging/wheel/build_from_source.sh + +IS_CONDA=1 python setup.py install --single-version-externally-managed --record=record.txt diff --git a/build_tools/packaging/conda/torchaudio/meta.yaml b/build_tools/packaging/conda/torchaudio/meta.yaml index 2e7d5d3944..fdf41e52a9 100644 --- a/build_tools/packaging/conda/torchaudio/meta.yaml +++ b/build_tools/packaging/conda/torchaudio/meta.yaml @@ -1,10 +1,9 @@ package: - name: torchaudio{{ environ.get('TORCHAUDIO_PACKAGE_SUFFIX') }} + name: torchaudio version: "{{ environ.get('TORCHAUDIO_BUILD_VERSION') }}" source: - git_rev: v{{ environ.get('TORCHAUDIO_BUILD_VERSION') }} - git_url: https://github.com/pytorch/audio.git + path: "{{ environ.get('TORCHAUDIO_GITHUB_ROOT_DIR') }}" requirements: build: @@ -14,20 +13,16 @@ requirements: host: - python - setuptools - - pytorch{{ environ.get('TORCHAUDIO_PACKAGE_SUFFIX') }} >=1.1.0 - - sox + - pytorch >=1.2.0 run: - python - - pytorch{{ environ.get('TORCHAUDIO_PACKAGE_SUFFIX') }} >=1.1.0 - - librosa >=0.4.3 - - scipy - - sox + - typing + - pytorch >=1.2.0 build: number: {{ environ.get('TORCHAUDIO_BUILD_NUMBER') }} string: py{{py}}_{{environ.get('TORCHAUDIO_BUILD_NUMBER')}} - script: IS_CONDA=1 python setup.py install --single-version-externally-managed --record=record.txt # [not win] test: imports: @@ -42,7 +37,8 @@ test: requires: - pytest - - librosa >=0.4.3 + # Ideally we would test this, but conda doesn't provide librosa + # - librosa >=0.4.3 - scipy about: diff --git a/build_tools/packaging/wheel/build_from_source.sh b/build_tools/packaging/wheel/build_from_source.sh index 31279ac4ed..eacd890852 100755 --- a/build_tools/packaging/wheel/build_from_source.sh +++ b/build_tools/packaging/wheel/build_from_source.sh @@ -1,6 +1,6 @@ rm -rf source_code mkdir source_code -cd source_code +pushd source_code wget -q -O sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" wget -q -O lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" @@ -15,13 +15,13 @@ tar xfp libmad-0.15.1b.tar.gz # build lame, statically pushd lame-3.99.5 -./configure --disable-shared --enable-static --prefix="$PREFIX/audio/third_party/lame" CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking --enable-nasm +./configure --disable-shared --enable-static --prefix="$PREFIX/third_party/lame" CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking --enable-nasm make -s -j && make install popd # build flac, statically pushd flac-1.3.2 -./configure --disable-shared --enable-static --prefix="$PREFIX/audio/third_party/flac" CFLAGS=-fPIC CXXFLAGS=-fPIC \ +./configure --disable-shared --enable-static --prefix="$PREFIX/third_party/flac" CFLAGS=-fPIC CXXFLAGS=-fPIC \ --with-pic --disable-debug --disable-dependency-tracking make -s -j && make install popd @@ -30,7 +30,7 @@ popd pushd libmad-0.15.1b # See https://stackoverflow.com/a/12864879/23845 sed -i.bak 's/-march=i486//' configure -./configure --disable-shared --enable-static --prefix="$PREFIX/audio/third_party/mad" CFLAGS=-fPIC CXXFLAGS=-fPIC \ +./configure --disable-shared --enable-static --prefix="$PREFIX/third_party/mad" CFLAGS=-fPIC CXXFLAGS=-fPIC \ --with-pic --disable-debug --disable-dependency-tracking make -s -j && make install popd @@ -40,9 +40,11 @@ popd # finds png and enables it. We don't want it; we'd need to package # it statically if we do. pushd sox-14.4.2 -./configure --disable-shared --enable-static --prefix="$PREFIX/audio/third_party/sox" \ - LDFLAGS="-L$PREFIX/audio/third_party/lame/lib -L$PREFIX/audio/third_party/flac/lib -L$PREFIX/audio/third_party/mad/lib" \ - CPPFLAGS="-I$PREFIX/audio/third_party/lame/include -I$PREFIX/audio/third_party/flac/include -I$PREFIX/audio/third_party/mad/include" \ +./configure --disable-shared --enable-static --prefix="$PREFIX/third_party/sox" \ + LDFLAGS="-L$PREFIX/third_party/lame/lib -L$PREFIX/third_party/flac/lib -L$PREFIX/third_party/mad/lib" \ + CPPFLAGS="-I$PREFIX/third_party/lame/include -I$PREFIX/third_party/flac/include -I$PREFIX/third_party/mad/include" \ --with-lame --with-flac --with-mad --without-png --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking make -s -j && make install popd + +popd diff --git a/build_tools/packaging/wheel/linux_manywheel.sh b/build_tools/packaging/wheel/linux_manywheel.sh index 9540a1ff5c..17be4c280c 100755 --- a/build_tools/packaging/wheel/linux_manywheel.sh +++ b/build_tools/packaging/wheel/linux_manywheel.sh @@ -21,7 +21,7 @@ rm -rf audio git clone https://github.com/pytorch/audio -b v${TORCHAUDIO_BUILD_VERSION} mkdir audio/third_party -export PREFIX="/tmp" +export PREFIX="/tmp/audio" . /remote/wheel/build_from_source.sh cd /tmp/audio diff --git a/build_tools/packaging/wheel/osx_wheel.sh b/build_tools/packaging/wheel/osx_wheel.sh index b018bb7ffe..9e23057861 100755 --- a/build_tools/packaging/wheel/osx_wheel.sh +++ b/build_tools/packaging/wheel/osx_wheel.sh @@ -26,7 +26,7 @@ rm -rf audio git clone https://github.com/pytorch/audio -b v${TORCHAUDIO_BUILD_VERSION} mkdir audio/third_party -export PREFIX="/tmp" +export PREFIX="/tmp/audio" . $CURR_PATH/build_from_source.sh cd /tmp/audio diff --git a/setup.py b/setup.py index 1a1e914a72..eea67ae8ec 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,10 @@ def check_env_flag(name, default=''): include_dirs = [] extra_objects = [] -if IS_WHEEL: +# Hypothetically, the conda distribution could rely on an external sox, +# but the library is pretty small and it is not available on the default +# anaconda channel. So we statically link it in, just as we do with wheels. +if IS_WHEEL or IS_CONDA: audio_path = os.path.dirname(os.path.abspath(__file__)) include_dirs += [os.path.join(audio_path, 'third_party/flac/include')] @@ -49,11 +52,6 @@ def check_env_flag(name, default=''): else: libraries += ['sox'] -if IS_CONDA: - # We want $PREFIX/include for conda (for sox.h) - lib_path = os.path.dirname(sys.executable) - include_dirs += [os.path.join(os.path.dirname(lib_path), 'include')] - # Creating the version file cwd = os.path.dirname(os.path.abspath(__file__)) From 34e1d3cfae414a5f53cb63bff0aee830a15bbdd2 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Thu, 8 Aug 2019 12:31:39 -0400 Subject: [PATCH 09/17] 0.3.0 packaging (#221) * Rewrite our packaging from scratch (#217) * New entry points are packaging/build_wheel.sh and packaging/build_conda.sh. The only mandatory environment variable you have to set is PYTHON_VERSION * CircleCI configuration uses 2.1-style parametrized builds to let you toggle python version, etc. as you do builds. We create a separate job per build configuration for maximum parallelism * build_tools/packaging got moved to packaging, to be in-line with directory structure in torchvision * The build_conda.sh and build_wheel.sh delegate most of the heavy lifting to pkg_helpers.bash, which defines a number of bash functions for performing common operations. The intent is that I'll copy-paste this file between other domain API projects. * TORCHAUDIO_ prefix removed from envvars, so that I can more easily share packaging scripts between projects * BUILD_VERSION is completely gone; just change the version number if you need to rebuild * No more logic for cloning and checking out a fresh copy of torchaudio * Setup cuda_suffix (#218) Signed-off-by: Edward Z. Yang * Add uploading support (#220) Also bugfixes and updates to the packaging scripts. * Finish up and set parameters Signed-off-by: Edward Z. Yang * Updates from vision Signed-off-by: Edward Z. Yang * Go time Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 308 ++++++++++++++++++ .circleci/config.yml.in | 207 ++++++++++++ .circleci/regenerate.py | 13 + build_tools/packaging/conda/build_audio.sh | 48 --- .../packaging/wheel/linux_manywheel.sh | 46 --- build_tools/packaging/wheel/osx_wheel.sh | 48 --- .../packaging => packaging}/README.md | 22 +- packaging/build_conda.sh | 11 + .../wheel => packaging}/build_from_source.sh | 24 +- packaging/build_wheel.sh | 14 + packaging/pkg_helpers.bash | 218 +++++++++++++ .../conda => packaging}/torchaudio/build.sh | 3 +- .../conda => packaging}/torchaudio/meta.yaml | 12 +- setup.py | 20 +- 14 files changed, 811 insertions(+), 183 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 .circleci/config.yml.in create mode 100755 .circleci/regenerate.py delete mode 100755 build_tools/packaging/conda/build_audio.sh delete mode 100755 build_tools/packaging/wheel/linux_manywheel.sh delete mode 100755 build_tools/packaging/wheel/osx_wheel.sh rename {build_tools/packaging => packaging}/README.md (77%) create mode 100755 packaging/build_conda.sh rename {build_tools/packaging/wheel => packaging}/build_from_source.sh (74%) create mode 100755 packaging/build_wheel.sh create mode 100644 packaging/pkg_helpers.bash rename {build_tools/packaging/conda => packaging}/torchaudio/build.sh (64%) rename {build_tools/packaging/conda => packaging}/torchaudio/meta.yaml (69%) diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..1d8cca2d9c --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,308 @@ +version: 2.1 + +# How to test the Linux jobs: +# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7 +# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test. +# Job names are 'name:' key. + +binary_common: &binary_common + parameters: + # Edit these defaults to do a release + build_version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "0.3.0" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "1.2.0" + # Don't edit these + python_version: + description: "Python version to build against (e.g., 3.7)" + type: string + unicode_abi: + description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" + type: string + default: "" + environment: + PYTHON_VERSION: << parameters.python_version >> + BUILD_VERSION: << parameters.build_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + UNICODE_ABI: << parameters.unicode_abi >> + CU_VERSION: cpu + +jobs: + circleci_consistency: + docker: + - image: circleci/python:3.7 + steps: + - checkout + - run: + command: | + pip install --user --progress-bar off jinja2 + python .circleci/regenerate.py + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + + binary_linux_wheel: + <<: *binary_common + docker: + - image: "soumith/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + + binary_linux_conda: + <<: *binary_common + docker: + - image: "soumith/conda-cuda" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_conda.sh + - store_artifacts: + path: /opt/conda/conda-bld/linux-64 + - persist_to_workspace: + root: /opt/conda/conda-bld/linux-64 + paths: + - "*" + + binary_macos_wheel: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + # Cannot easily deduplicate this as source'ing activate + # will set environment variables which we need to propagate + # to build_wheel.sh + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + + binary_macos_conda: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build + packaging/build_conda.sh + - store_artifacts: + path: /Users/distiller/miniconda3/conda-bld/osx-64 + - persist_to_workspace: + root: /Users/distiller/miniconda3/conda-bld/osx-64 + paths: + - "*" + + # Requires org-member context + binary_conda_upload: + docker: + - image: continuumio/miniconda + steps: + - attach_workspace: + at: ~/workspace + - run: + command: | + # Prevent credential from leaking + conda install -yq anaconda-client + set +x + anaconda login \ + --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \ + --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD" + set -x + anaconda upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force + + # Requires org-member context + binary_wheel_upload: + docker: + - image: circleci/python:3.7 + steps: + - attach_workspace: + at: ~/workspace + - checkout + - run: + command: | + pip install --user awscli + export PATH="$HOME/.local/bin:$PATH" + # Prevent credential from leaking + set +x + export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" + export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" + set -x + for pkg in ~/workspace/*.whl; do + aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read + done + +workflows: + build: + jobs: + - circleci_consistency + + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py2.7 + python_version: "2.7" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py2.7_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py2.7 + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py2.7_unicode_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py2.7_unicode + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py3.5 + python_version: "3.5" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py3.5_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py3.5 + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py3.6 + python_version: "3.6" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py3.6_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py3.6 + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py3.7 + python_version: "3.7" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py3.7_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py3.7 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py2.7 + python_version: "2.7" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py2.7_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py2.7 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py2.7_unicode_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py2.7_unicode + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py3.5 + python_version: "3.5" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py3.5_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py3.5 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py3.6 + python_version: "3.6" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py3.6_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py3.6 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py3.7 + python_version: "3.7" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py3.7_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py3.7 + - binary_linux_conda: + name: nightly_binary_linux_conda_py2.7 + python_version: "2.7" + - binary_conda_upload: + name: nightly_binary_linux_conda_py2.7_upload + context: org-member + requires: + - nightly_binary_linux_conda_py2.7 + - binary_linux_conda: + name: nightly_binary_linux_conda_py3.5 + python_version: "3.5" + - binary_conda_upload: + name: nightly_binary_linux_conda_py3.5_upload + context: org-member + requires: + - nightly_binary_linux_conda_py3.5 + - binary_linux_conda: + name: nightly_binary_linux_conda_py3.6 + python_version: "3.6" + - binary_conda_upload: + name: nightly_binary_linux_conda_py3.6_upload + context: org-member + requires: + - nightly_binary_linux_conda_py3.6 + - binary_linux_conda: + name: nightly_binary_linux_conda_py3.7 + python_version: "3.7" + - binary_conda_upload: + name: nightly_binary_linux_conda_py3.7_upload + context: org-member + requires: + - nightly_binary_linux_conda_py3.7 + - binary_macos_conda: + name: nightly_binary_macos_conda_py2.7 + python_version: "2.7" + - binary_conda_upload: + name: nightly_binary_macos_conda_py2.7_upload + context: org-member + requires: + - nightly_binary_macos_conda_py2.7 + - binary_macos_conda: + name: nightly_binary_macos_conda_py3.5 + python_version: "3.5" + - binary_conda_upload: + name: nightly_binary_macos_conda_py3.5_upload + context: org-member + requires: + - nightly_binary_macos_conda_py3.5 + - binary_macos_conda: + name: nightly_binary_macos_conda_py3.6 + python_version: "3.6" + - binary_conda_upload: + name: nightly_binary_macos_conda_py3.6_upload + context: org-member + requires: + - nightly_binary_macos_conda_py3.6 + - binary_macos_conda: + name: nightly_binary_macos_conda_py3.7 + python_version: "3.7" + - binary_conda_upload: + name: nightly_binary_macos_conda_py3.7_upload + context: org-member + requires: + - nightly_binary_macos_conda_py3.7 diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in new file mode 100644 index 0000000000..76468db6ba --- /dev/null +++ b/.circleci/config.yml.in @@ -0,0 +1,207 @@ +version: 2.1 + +# How to test the Linux jobs: +# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7 +# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test. +# Job names are 'name:' key. + +binary_common: &binary_common + parameters: + # Edit these defaults to do a release + build_version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "0.3.0" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "1.2.0" + # Don't edit these + python_version: + description: "Python version to build against (e.g., 3.7)" + type: string + unicode_abi: + description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" + type: string + default: "" + environment: + PYTHON_VERSION: << parameters.python_version >> + BUILD_VERSION: << parameters.build_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + UNICODE_ABI: << parameters.unicode_abi >> + CU_VERSION: cpu + +jobs: + circleci_consistency: + docker: + - image: circleci/python:3.7 + steps: + - checkout + - run: + command: | + pip install --user --progress-bar off jinja2 + python .circleci/regenerate.py + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + + binary_linux_wheel: + <<: *binary_common + docker: + - image: "soumith/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + + binary_linux_conda: + <<: *binary_common + docker: + - image: "soumith/conda-cuda" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_conda.sh + - store_artifacts: + path: /opt/conda/conda-bld/linux-64 + - persist_to_workspace: + root: /opt/conda/conda-bld/linux-64 + paths: + - "*" + + binary_macos_wheel: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + # Cannot easily deduplicate this as source'ing activate + # will set environment variables which we need to propagate + # to build_wheel.sh + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + + binary_macos_conda: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build + packaging/build_conda.sh + - store_artifacts: + path: /Users/distiller/miniconda3/conda-bld/osx-64 + - persist_to_workspace: + root: /Users/distiller/miniconda3/conda-bld/osx-64 + paths: + - "*" + + # Requires org-member context + binary_conda_upload: + docker: + - image: continuumio/miniconda + steps: + - attach_workspace: + at: ~/workspace + - run: + command: | + # Prevent credential from leaking + conda install -yq anaconda-client + set +x + anaconda login \ + --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \ + --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD" + set -x + anaconda upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force + + # Requires org-member context + binary_wheel_upload: + docker: + - image: circleci/python:3.7 + steps: + - attach_workspace: + at: ~/workspace + - checkout + - run: + command: | + pip install --user awscli + export PATH="$HOME/.local/bin:$PATH" + # Prevent credential from leaking + set +x + export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" + export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" + set -x + for pkg in ~/workspace/*.whl; do + aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read + done + + +{%- macro workflow(btype, os, python_version, unicode, prefix='', upload=False) %} + - binary_{{os}}_{{btype}}: + name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }} + python_version: "{{python_version}}" +{%- if unicode %} + unicode_abi: "1" +{%- endif %} + +{%- if upload %} + - binary_{{btype}}_upload: + name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }}_upload + context: org-member + requires: + - {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }} +{%- endif %} +{%- endmacro %} + +{%- macro workflows(prefix='', upload=False) %} +{%- for btype in ["wheel", "conda"] -%} +{%- for os in ["linux", "macos"] -%} +{%- for python_version in ["2.7", "3.5", "3.6", "3.7"] -%} +{%- for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]) -%} + {{ workflow(btype, os, python_version, unicode, prefix=prefix, upload=upload) }} +{%- endfor -%} +{%- endfor -%} +{%- endfor -%} +{%- endfor %} +{%- endmacro %} + +workflows: + build: +{%- if False %} + jobs: + - circleci_consistency + {{ workflows() }} + + nightly: + triggers: + - schedule: + cron: "0 9 * * *" + filters: + branches: + only: + - master +{%- endif %} + jobs: + - circleci_consistency + {{ workflows(prefix="nightly_", upload=True) }} + diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py new file mode 100755 index 0000000000..dcc3fb2c23 --- /dev/null +++ b/.circleci/regenerate.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +import jinja2 +import os.path + +d = os.path.dirname(__file__) +env = jinja2.Environment( + loader=jinja2.FileSystemLoader(d), + lstrip_blocks=True, + autoescape=False, +) +with open(os.path.join(d, 'config.yml'), 'w') as f: + f.write(env.get_template('config.yml.in').render()) diff --git a/build_tools/packaging/conda/build_audio.sh b/build_tools/packaging/conda/build_audio.sh deleted file mode 100755 index 98518200d7..0000000000 --- a/build_tools/packaging/conda/build_audio.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -if [[ -x "/remote/anaconda_token" ]]; then - . /remote/anaconda_token || true -fi - -set -ex - - # Function to retry functions that sometimes timeout or have flaky failures -retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) -} - -export TORCHAUDIO_BUILD_VERSION="0.3.0" -export TORCHAUDIO_BUILD_NUMBER=1 - -SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -audio_rootdir="$(pwd)/torchaudio-src" - -if [[ "$BRANCH" == "" ]]; then - BRANCH=master -fi - -if [[ ! -d "$audio_rootdir" ]]; then - rm -rf "$audio_rootdir" - git clone "https://github.com/pytorch/audio" "$audio_rootdir" -b "$BRANCH" -fi - -export TORCHAUDIO_GITHUB_ROOT_DIR="$audio_rootdir" - -cd "$SOURCE_DIR" - -ANACONDA_USER=pytorch -conda config --set anaconda_upload no - -export CONDA_CUDATOOLKIT_CONSTRAINT="" -export CUDA_VERSION="None" -if [[ "$OSTYPE" == "darwin"* ]]; then - export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ -fi - -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 torchaudio -# Currently disabled, see https://github.com/pytorch/pytorch/issues/23738 -# time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchaudio -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 torchaudio -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 torchaudio - - set +e diff --git a/build_tools/packaging/wheel/linux_manywheel.sh b/build_tools/packaging/wheel/linux_manywheel.sh deleted file mode 100755 index 17be4c280c..0000000000 --- a/build_tools/packaging/wheel/linux_manywheel.sh +++ /dev/null @@ -1,46 +0,0 @@ -if [ "$#" -ne 1 ]; then - echo "Illegal number of parameters. Pass cuda version" - echo "CUDA version should be cu90, cu100 or cpu" - exit 1 -fi -export CUVER="$1" # cu90 cu100 cpu - -export TORCHAUDIO_BUILD_VERSION="0.3.0" -export TORCHAUDIO_BUILD_NUMBER="1" -export OUT_DIR="/remote/$CUVER" - -cd /opt/python -DESIRED_PYTHON=(*/) -for desired_py in "${DESIRED_PYTHON[@]}"; do - python_installations+=("/opt/python/$desired_py") -done - -OLD_PATH=$PATH -cd /tmp -rm -rf audio -git clone https://github.com/pytorch/audio -b v${TORCHAUDIO_BUILD_VERSION} -mkdir audio/third_party - -export PREFIX="/tmp/audio" -. /remote/wheel/build_from_source.sh - -cd /tmp/audio - -for PYDIR in "${python_installations[@]}"; do - export PATH=$PYDIR/bin:$OLD_PATH - pip install --upgrade pip - - # For true hermetic builds, you ought to be constructing the docker - # from scratch each time. But this makes things marginally safer if - # you aren't doing this. - pip uninstall -y torch || true - pip uninstall -y torch_nightly || true - - # NB: do not actually install requirements.txt; that is only needed for - # testing - pip install torch numpy future - IS_WHEEL=1 python setup.py clean - IS_WHEEL=1 python setup.py bdist_wheel - mkdir -p $OUT_DIR - cp dist/*.whl $OUT_DIR/ -done diff --git a/build_tools/packaging/wheel/osx_wheel.sh b/build_tools/packaging/wheel/osx_wheel.sh deleted file mode 100755 index 9e23057861..0000000000 --- a/build_tools/packaging/wheel/osx_wheel.sh +++ /dev/null @@ -1,48 +0,0 @@ -if [[ ":$PATH:" == *"conda"* ]]; then - echo "existing anaconda install in PATH, remove it and run script" - exit 1 -fi -# download and activate anaconda -rm -rf ~/minconda_wheel_env_tmp -wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh && \ - chmod +x Miniconda3-latest-MacOSX-x86_64.sh && \ - ./Miniconda3-latest-MacOSX-x86_64.sh -b -p ~/minconda_wheel_env_tmp && \ - rm Miniconda3-latest-MacOSX-x86_64.sh - -. ~/minconda_wheel_env_tmp/bin/activate - -export TORCHAUDIO_BUILD_VERSION="0.3.0" -export TORCHAUDIO_BUILD_NUMBER="1" -export OUT_DIR=~/torchaudio_wheels - -export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ - -# TODO remove when pytorch is good https://github.com/pytorch/pytorch/issues/20030 -brew install libomp -CURR_PATH=$(pwd) - -cd /tmp -rm -rf audio -git clone https://github.com/pytorch/audio -b v${TORCHAUDIO_BUILD_VERSION} -mkdir audio/third_party - -export PREFIX="/tmp/audio" -. $CURR_PATH/build_from_source.sh - -cd /tmp/audio - -desired_pythons=( "2.7" "3.5" "3.6" "3.7" ) -# for each python -for desired_python in "${desired_pythons[@]}" -do - # create and activate python env - env_name="env$desired_python" - conda create -yn $env_name python="$desired_python" - conda activate $env_name - - pip install torch numpy future - IS_WHEEL=1 python setup.py clean - IS_WHEEL=1 python setup.py bdist_wheel - mkdir -p $OUT_DIR - cp dist/*.whl $OUT_DIR/ -done diff --git a/build_tools/packaging/README.md b/packaging/README.md similarity index 77% rename from build_tools/packaging/README.md rename to packaging/README.md index 6d2efc6634..f3ee62e62c 100644 --- a/build_tools/packaging/README.md +++ b/packaging/README.md @@ -5,12 +5,9 @@ ### Linux ```bash -# NB: We don't actually need CUDA; you don't have to run this on nvidia-docker -nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash -pushd remote/conda -conda config --add channels pytorch -conda config --add channels conda-forge -./build_audio.sh +docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash +cd remote +PYTHON_VERSION=3.7 packaging/build_conda.sh ``` To install bz2, @@ -32,11 +29,7 @@ anaconda upload -u pytorch /opt/conda/conda-bld/linux-64/torchaudio*.bz2 ```bash # create a fresh anaconda environment / install and activate it -cd packaging/conda -conda install -y conda-build anaconda-client -conda config --add channels pytorch -conda config --add channels conda-forge -./build_audio.sh +PYTHON_VERSION=3.7 packaging/build_conda.sh ``` To install bz2, @@ -64,8 +57,8 @@ anaconda upload -u pytorch /Users/jamarshon/anaconda3/conda-bld/osx-64/torchaudi ```bash nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/manylinux-cuda90:latest bash -cd remote/wheel -./linux_manywheel.sh cpu +cd remote +PYTHON_VERSION=3.7 packaging/build_wheel.sh ``` To install wheels, @@ -84,8 +77,7 @@ cd ../cpu ### OSX ```bash -pushd wheel -./osx_wheel.sh +PYTHON_VERSION=3.7 packaging/build_wheel.sh ``` To install wheels, diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh new file mode 100755 index 0000000000..be00e74f3c --- /dev/null +++ b/packaging/build_conda.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -ex + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +. "$script_dir/pkg_helpers.bash" + +export NO_CUDA_PACKAGE=1 +setup_env 0.4.0 +export SOURCE_ROOT_DIR="$PWD" +setup_conda_pytorch_constraint +conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchaudio diff --git a/build_tools/packaging/wheel/build_from_source.sh b/packaging/build_from_source.sh similarity index 74% rename from build_tools/packaging/wheel/build_from_source.sh rename to packaging/build_from_source.sh index eacd890852..3e16da5ede 100755 --- a/build_tools/packaging/wheel/build_from_source.sh +++ b/packaging/build_from_source.sh @@ -1,11 +1,19 @@ -rm -rf source_code -mkdir source_code -pushd source_code +#!/bin/bash -wget -q -O sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" -wget -q -O lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" -wget -q -O flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" -wget -q -O libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" +set -ex + +# Arguments: PREFIX, specifying where to install dependencies into + +PREFIX="$1" + +rm -rf /tmp/torchaudio-deps +mkdir /tmp/torchaudio-deps +pushd /tmp/torchaudio-deps + +curl -L -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" +curl -L -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" +curl -L -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" +curl -L -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" # unpack the dependencies tar xfp sox-14.4.2.tar.bz2 @@ -43,7 +51,7 @@ pushd sox-14.4.2 ./configure --disable-shared --enable-static --prefix="$PREFIX/third_party/sox" \ LDFLAGS="-L$PREFIX/third_party/lame/lib -L$PREFIX/third_party/flac/lib -L$PREFIX/third_party/mad/lib" \ CPPFLAGS="-I$PREFIX/third_party/lame/include -I$PREFIX/third_party/flac/include -I$PREFIX/third_party/mad/include" \ - --with-lame --with-flac --with-mad --without-png --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking + --with-lame --with-flac --with-mad --without-coreaudio --without-png --without-oggvorbis --without-oss --without-sndfile CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking make -s -j && make install popd diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh new file mode 100755 index 0000000000..60818f336b --- /dev/null +++ b/packaging/build_wheel.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -ex + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +. "$script_dir/pkg_helpers.bash" + +export NO_CUDA_PACKAGE=1 +setup_env 0.4.0 +setup_wheel_python +"$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies +pip_install numpy future +setup_pip_pytorch_version +python setup.py clean +IS_WHEEL=1 python setup.py bdist_wheel diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash new file mode 100644 index 0000000000..461cc980b9 --- /dev/null +++ b/packaging/pkg_helpers.bash @@ -0,0 +1,218 @@ +# A set of useful bash functions for common functionality we need to do in +# many build scripts + + +# Setup CUDA environment variables, based on CU_VERSION +# +# Inputs: +# CU_VERSION (cpu, cu92, cu100) +# NO_CUDA_PACKAGE (bool) +# +# Outputs: +# VERSION_SUFFIX (e.g., "") +# PYTORCH_VERSION_SUFFIX (e.g., +cpu) +# WHEEL_DIR (e.g., cu100/) +# CUDA_HOME (e.g., /usr/local/cuda-9.2, respected by torch.utils.cpp_extension) +# FORCE_CUDA (respected by torchvision setup.py) +# NVCC_FLAGS (respected by torchvision setup.py) +# +# Precondition: CUDA versions are installed in their conventional locations in +# /usr/local/cuda-* +# +# NOTE: Why VERSION_SUFFIX versus PYTORCH_VERSION_SUFFIX? If you're building +# a package with CUDA on a platform we support CUDA on, VERSION_SUFFIX == +# PYTORCH_VERSION_SUFFIX and everyone is happy. However, if you are building a +# package with only CPU bits (e.g., torchaudio), then VERSION_SUFFIX is always +# empty, but PYTORCH_VERSION_SUFFIX is +cpu (because that's how you get a CPU +# version of a Python package. But that doesn't apply if you're on OS X, +# since the default CU_VERSION on OS X is cpu. +setup_cuda() { + if [[ "$(uname)" == Darwin ]] || [[ -n "$NO_CUDA_PACKAGE" ]]; then + if [[ "$CU_VERSION" != "cpu" ]]; then + echo "CU_VERSION on OS X / package with no CUDA must be cpu" + exit 1 + fi + if [[ "$(uname)" == Darwin ]]; then + export PYTORCH_VERSION_SUFFIX="" + else + export PYTORCH_VERSION_SUFFIX="+cpu" + fi + export VERSION_SUFFIX="" + # NB: When there is no CUDA package available, we put these + # packages in the top-level directory, so they are eligible + # for selection even if you are otherwise trying to install + # a cu100 stack. This differs from when there ARE CUDA packages + # available; then we don't want the cpu package; we want + # to give you as much goodies as possible. + export WHEEL_DIR="" + else + case "$CU_VERSION" in + cu100) + export PYTORCH_VERSION_SUFFIX="" + export CUDA_HOME=/usr/local/cuda-10.0/ + export FORCE_CUDA=1 + # Hard-coding gencode flags is temporary situation until + # https://github.com/pytorch/pytorch/pull/23408 lands + export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50" + ;; + cu92) + export CUDA_HOME=/usr/local/cuda-9.2/ + export PYTORCH_VERSION_SUFFIX="+cu92" + export FORCE_CUDA=1 + export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50" + ;; + cpu) + export PYTORCH_VERSION_SUFFIX="+cpu" + ;; + *) + echo "Unrecognized CU_VERSION=$CU_VERSION" + esac + export VERSION_SUFFIX="$PYTORCH_VERSION_SUFFIX" + export WHEEL_DIR="$CU_VERSION/" + fi +} + +# Populate build version if necessary, and add version suffix +# +# Inputs: +# BUILD_VERSION (e.g., 0.2.0 or empty) +# +# Outputs: +# BUILD_VERSION (e.g., 0.2.0.dev20190807+cpu) +# +# Fill BUILD_VERSION if it doesn't exist already with a nightly string +# Usage: setup_build_version 0.2.0 +setup_build_version() { + if [[ -z "$BUILD_VERSION" ]]; then + export BUILD_VERSION="$1.dev$(date "+%Y%m%d")$VERSION_SUFFIX" + else + export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" + fi +} + +# Set some useful variables for OS X, if applicable +setup_macos() { + if [[ "$(uname)" == Darwin ]]; then + export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ + fi +} + +# Top-level entry point for things every package will need to do +# +# Usage: setup_env 0.2.0 +setup_env() { + setup_cuda + setup_build_version "$1" + setup_macos +} + +# Function to retry functions that sometimes timeout or have flaky failures +retry () { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) +} + +# Inputs: +# PYTHON_VERSION (2.7, 3.5, 3.6, 3.7) +# UNICODE_ABI (bool) +# +# Outputs: +# PATH modified to put correct Python version in PATH +# +# Precondition: If Linux, you are in a soumith/manylinux-cuda* Docker image +setup_wheel_python() { + if [[ "$(uname)" == Darwin ]]; then + eval "$(conda shell.bash hook)" + conda env remove -n "env$PYTHON_VERSION" || true + conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" + conda activate "env$PYTHON_VERSION" + else + case "$PYTHON_VERSION" in + 2.7) + if [[ -n "$UNICODE_ABI" ]]; then + python_abi=cp27-cp27mu + else + python_abi=cp27-cp27m + fi + ;; + 3.5) python_abi=cp35-cp35m ;; + 3.6) python_abi=cp36-cp36m ;; + 3.7) python_abi=cp37-cp37m ;; + *) + echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" + exit 1 + ;; + esac + export PATH="/opt/python/$python_abi/bin:$PATH" + fi +} + +# Install with pip a bit more robustly than the default +pip_install() { + retry pip install --progress-bar off "$@" +} + +# Install torch with pip, respecting PYTORCH_VERSION, and record the installed +# version into PYTORCH_VERSION, if applicable +setup_pip_pytorch_version() { + if [[ -z "$PYTORCH_VERSION" ]]; then + # Install latest prerelease version of torch, per our nightlies, consistent + # with the requested cuda version + pip_install --pre torch -f "https://download.pytorch.org/whl/nightly/${WHEEL_DIR}torch_nightly.html" + if [[ "$CUDA_VERSION" == "cpu" ]]; then + # CUDA and CPU are ABI compatible on the CPU-only parts, so strip + # in this case + export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" + else + export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//')" + fi + else + pip_install "torch==$PYTORCH_VERSION$CUDA_SUFFIX" \ + -f https://download.pytorch.org/whl/torch_stable.html \ + -f https://download.pytorch.org/whl/nightly/torch_nightly.html + fi +} + +# Fill PYTORCH_VERSION with the latest conda nightly version, and +# CONDA_CHANNEL_FLAGS with appropriate flags to retrieve these versions +# +# You MUST have populated CUDA_SUFFIX before hand. +setup_conda_pytorch_constraint() { + if [[ -z "$PYTORCH_VERSION" ]]; then + export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" + export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" + else + export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly" + fi + if [[ "$CU_VERSION" == cpu ]]; then + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}" + export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION" + else + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${PYTORCH_VERSION_SUFFIX}" + export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${PYTORCH_VERSION_SUFFIX}" + fi +} + +# Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT +setup_conda_cudatoolkit_constraint() { + export CONDA_CPUONLY_FEATURE="" + if [[ "$(uname)" == Darwin ]]; then + export CONDA_CUDATOOLKIT_CONSTRAINT="" + else + case "$CU_VERSION" in + cu100) + export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.0,<10.1 # [not osx]" + ;; + cu92) + export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.2,<9.3 # [not osx]" + ;; + cpu) + export CONDA_CUDATOOLKIT_CONSTRAINT="" + export CONDA_CPUONLY_FEATURE="- cpuonly" + ;; + *) + echo "Unrecognized CU_VERSION=$CU_VERSION" + exit 1 + ;; + esac + fi +} diff --git a/build_tools/packaging/conda/torchaudio/build.sh b/packaging/torchaudio/build.sh similarity index 64% rename from build_tools/packaging/conda/torchaudio/build.sh rename to packaging/torchaudio/build.sh index 7e5917060f..dff89cf31b 100644 --- a/build_tools/packaging/conda/torchaudio/build.sh +++ b/packaging/torchaudio/build.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -ex -PREFIX="$(pwd)" -. build_tools/packaging/wheel/build_from_source.sh +packaging/build_from_source.sh "$(pwd)" IS_CONDA=1 python setup.py install --single-version-externally-managed --record=record.txt diff --git a/build_tools/packaging/conda/torchaudio/meta.yaml b/packaging/torchaudio/meta.yaml similarity index 69% rename from build_tools/packaging/conda/torchaudio/meta.yaml rename to packaging/torchaudio/meta.yaml index fdf41e52a9..0b89969fa2 100644 --- a/build_tools/packaging/conda/torchaudio/meta.yaml +++ b/packaging/torchaudio/meta.yaml @@ -1,9 +1,9 @@ package: name: torchaudio - version: "{{ environ.get('TORCHAUDIO_BUILD_VERSION') }}" + version: "{{ environ.get('BUILD_VERSION') }}" source: - path: "{{ environ.get('TORCHAUDIO_GITHUB_ROOT_DIR') }}" + path: "{{ environ.get('SOURCE_ROOT_DIR') }}" requirements: build: @@ -13,16 +13,16 @@ requirements: host: - python - setuptools - - pytorch >=1.2.0 + - cpuonly + {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }} run: - python - typing - - pytorch >=1.2.0 + {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} build: - number: {{ environ.get('TORCHAUDIO_BUILD_NUMBER') }} - string: py{{py}}_{{environ.get('TORCHAUDIO_BUILD_NUMBER')}} + string: py{{py}} test: imports: diff --git a/setup.py b/setup.py index eea67ae8ec..08eb3fb792 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ def check_env_flag(name, default=''): # Creating the version file cwd = os.path.dirname(os.path.abspath(__file__)) -version = '0.3.0a0' +version = '0.3.0' sha = 'Unknown' try: @@ -63,12 +63,8 @@ def check_env_flag(name, default=''): except Exception: pass -if os.getenv('TORCHAUDIO_BUILD_VERSION'): - assert os.getenv('TORCHAUDIO_BUILD_NUMBER') is not None - build_number = int(os.getenv('TORCHAUDIO_BUILD_NUMBER')) - version = os.getenv('TORCHAUDIO_BUILD_VERSION') - if build_number > 1: - version += '.post' + str(build_number) +if os.getenv('BUILD_VERSION'): + version = os.getenv('BUILD_VERSION') elif sha != 'Unknown': version += '+' + sha[:7] print('-- Building version ' + version) @@ -78,11 +74,15 @@ def check_env_flag(name, default=''): f.write("__version__ = '{}'\n".format(version)) f.write("git_version = {}\n".format(repr(sha))) -pytorch_package_name = os.getenv('TORCHAUDIO_PYTORCH_DEPENDENCY_NAME', 'torch') +pytorch_package_version = os.getenv('PYTORCH_VERSION') + +pytorch_package_dep = 'torch' +if pytorch_package_version is not None: + pytorch_package_dep += "==" + pytorch_package_version setup( name="torchaudio", - version="0.2", + version=version, description="An audio package for PyTorch", url="https://github.com/pytorch/audio", author="Soumith Chintala, David Pollack, Sean Naren, Peter Goldsborough", @@ -115,5 +115,5 @@ def check_env_flag(name, default=''): extra_link_args=ela), ], cmdclass={'build_ext': BuildExtension}, - install_requires=[pytorch_package_name] + install_requires=[pytorch_package_dep] ) From 20001e4bac68f280c23c6381b01d1d8e1bfc34f2 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Thu, 8 Aug 2019 15:33:35 -0400 Subject: [PATCH 10/17] Don't build nightlies on 0.3.0 branch Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 127 ++++++---------------------------------- .circleci/config.yml.in | 16 +---- 2 files changed, 20 insertions(+), 123 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d8cca2d9c..cc15b07ec1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -161,148 +161,59 @@ workflows: - circleci_consistency - binary_linux_wheel: - name: nightly_binary_linux_wheel_py2.7 + name: binary_linux_wheel_py2.7 python_version: "2.7" - - binary_wheel_upload: - name: nightly_binary_linux_wheel_py2.7_upload - context: org-member - requires: - - nightly_binary_linux_wheel_py2.7 - binary_linux_wheel: - name: nightly_binary_linux_wheel_py2.7_unicode + name: binary_linux_wheel_py2.7_unicode python_version: "2.7" unicode_abi: "1" - - binary_wheel_upload: - name: nightly_binary_linux_wheel_py2.7_unicode_upload - context: org-member - requires: - - nightly_binary_linux_wheel_py2.7_unicode - binary_linux_wheel: - name: nightly_binary_linux_wheel_py3.5 + name: binary_linux_wheel_py3.5 python_version: "3.5" - - binary_wheel_upload: - name: nightly_binary_linux_wheel_py3.5_upload - context: org-member - requires: - - nightly_binary_linux_wheel_py3.5 - binary_linux_wheel: - name: nightly_binary_linux_wheel_py3.6 + name: binary_linux_wheel_py3.6 python_version: "3.6" - - binary_wheel_upload: - name: nightly_binary_linux_wheel_py3.6_upload - context: org-member - requires: - - nightly_binary_linux_wheel_py3.6 - binary_linux_wheel: - name: nightly_binary_linux_wheel_py3.7 + name: binary_linux_wheel_py3.7 python_version: "3.7" - - binary_wheel_upload: - name: nightly_binary_linux_wheel_py3.7_upload - context: org-member - requires: - - nightly_binary_linux_wheel_py3.7 - binary_macos_wheel: - name: nightly_binary_macos_wheel_py2.7 + name: binary_macos_wheel_py2.7 python_version: "2.7" - - binary_wheel_upload: - name: nightly_binary_macos_wheel_py2.7_upload - context: org-member - requires: - - nightly_binary_macos_wheel_py2.7 - binary_macos_wheel: - name: nightly_binary_macos_wheel_py2.7_unicode + name: binary_macos_wheel_py2.7_unicode python_version: "2.7" unicode_abi: "1" - - binary_wheel_upload: - name: nightly_binary_macos_wheel_py2.7_unicode_upload - context: org-member - requires: - - nightly_binary_macos_wheel_py2.7_unicode - binary_macos_wheel: - name: nightly_binary_macos_wheel_py3.5 + name: binary_macos_wheel_py3.5 python_version: "3.5" - - binary_wheel_upload: - name: nightly_binary_macos_wheel_py3.5_upload - context: org-member - requires: - - nightly_binary_macos_wheel_py3.5 - binary_macos_wheel: - name: nightly_binary_macos_wheel_py3.6 + name: binary_macos_wheel_py3.6 python_version: "3.6" - - binary_wheel_upload: - name: nightly_binary_macos_wheel_py3.6_upload - context: org-member - requires: - - nightly_binary_macos_wheel_py3.6 - binary_macos_wheel: - name: nightly_binary_macos_wheel_py3.7 + name: binary_macos_wheel_py3.7 python_version: "3.7" - - binary_wheel_upload: - name: nightly_binary_macos_wheel_py3.7_upload - context: org-member - requires: - - nightly_binary_macos_wheel_py3.7 - binary_linux_conda: - name: nightly_binary_linux_conda_py2.7 + name: binary_linux_conda_py2.7 python_version: "2.7" - - binary_conda_upload: - name: nightly_binary_linux_conda_py2.7_upload - context: org-member - requires: - - nightly_binary_linux_conda_py2.7 - binary_linux_conda: - name: nightly_binary_linux_conda_py3.5 + name: binary_linux_conda_py3.5 python_version: "3.5" - - binary_conda_upload: - name: nightly_binary_linux_conda_py3.5_upload - context: org-member - requires: - - nightly_binary_linux_conda_py3.5 - binary_linux_conda: - name: nightly_binary_linux_conda_py3.6 + name: binary_linux_conda_py3.6 python_version: "3.6" - - binary_conda_upload: - name: nightly_binary_linux_conda_py3.6_upload - context: org-member - requires: - - nightly_binary_linux_conda_py3.6 - binary_linux_conda: - name: nightly_binary_linux_conda_py3.7 + name: binary_linux_conda_py3.7 python_version: "3.7" - - binary_conda_upload: - name: nightly_binary_linux_conda_py3.7_upload - context: org-member - requires: - - nightly_binary_linux_conda_py3.7 - binary_macos_conda: - name: nightly_binary_macos_conda_py2.7 + name: binary_macos_conda_py2.7 python_version: "2.7" - - binary_conda_upload: - name: nightly_binary_macos_conda_py2.7_upload - context: org-member - requires: - - nightly_binary_macos_conda_py2.7 - binary_macos_conda: - name: nightly_binary_macos_conda_py3.5 + name: binary_macos_conda_py3.5 python_version: "3.5" - - binary_conda_upload: - name: nightly_binary_macos_conda_py3.5_upload - context: org-member - requires: - - nightly_binary_macos_conda_py3.5 - binary_macos_conda: - name: nightly_binary_macos_conda_py3.6 + name: binary_macos_conda_py3.6 python_version: "3.6" - - binary_conda_upload: - name: nightly_binary_macos_conda_py3.6_upload - context: org-member - requires: - - nightly_binary_macos_conda_py3.6 - binary_macos_conda: - name: nightly_binary_macos_conda_py3.7 + name: binary_macos_conda_py3.7 python_version: "3.7" - - binary_conda_upload: - name: nightly_binary_macos_conda_py3.7_upload - context: org-member - requires: - - nightly_binary_macos_conda_py3.7 + # workflows(prefix="nightly_", upload=True) \ No newline at end of file diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 76468db6ba..db1fa083d7 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -187,21 +187,7 @@ jobs: workflows: build: -{%- if False %} jobs: - circleci_consistency {{ workflows() }} - - nightly: - triggers: - - schedule: - cron: "0 9 * * *" - filters: - branches: - only: - - master -{%- endif %} - jobs: - - circleci_consistency - {{ workflows(prefix="nightly_", upload=True) }} - + # workflows(prefix="nightly_", upload=True) From 8557201094c8e8eb97fff3f0ef41aac12e43ac78 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Thu, 8 Aug 2019 17:05:18 -0400 Subject: [PATCH 11/17] Refactor version suffix so conda packages don't get suffixes. (#223) (#225) * Refactor version suffix so conda packages don't get suffixes. Signed-off-by: Edward Z. Yang * typofix Signed-off-by: Edward Z. Yang --- packaging/build_conda.sh | 1 + packaging/build_wheel.sh | 1 + packaging/pkg_helpers.bash | 82 +++++++++++++++++++------------------- 3 files changed, 44 insertions(+), 40 deletions(-) diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index be00e74f3c..47b428e773 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -4,6 +4,7 @@ set -ex script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" +export BUILD_TYPE="conda" export NO_CUDA_PACKAGE=1 setup_env 0.4.0 export SOURCE_ROOT_DIR="$PWD" diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 60818f336b..00def0928e 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -4,6 +4,7 @@ set -ex script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" . "$script_dir/pkg_helpers.bash" +export BUILD_TYPE="wheel" export NO_CUDA_PACKAGE=1 setup_env 0.4.0 setup_wheel_python diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 461cc980b9..bd9cbedcde 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -7,6 +7,7 @@ # Inputs: # CU_VERSION (cpu, cu92, cu100) # NO_CUDA_PACKAGE (bool) +# BUILD_TYPE (conda, wheel) # # Outputs: # VERSION_SUFFIX (e.g., "") @@ -27,55 +28,56 @@ # version of a Python package. But that doesn't apply if you're on OS X, # since the default CU_VERSION on OS X is cpu. setup_cuda() { - if [[ "$(uname)" == Darwin ]] || [[ -n "$NO_CUDA_PACKAGE" ]]; then - if [[ "$CU_VERSION" != "cpu" ]]; then - echo "CU_VERSION on OS X / package with no CUDA must be cpu" - exit 1 + + # First, compute version suffixes. By default, assume no version suffixes + export VERSION_SUFFIX="" + export PYTORCH_VERSION_SUFFIX="" + export WHEEL_DIR="" + # Wheel builds need suffixes (but not if they're on OS X, which never has suffix) + if [[ "$BUILD_TYPE" == "wheel" ]] && [[ "$(uname)" != Darwin ]]; then + # The default CUDA has no suffix + if [[ "$CU_VERSION" != "cu100" ]]; then + export PYTORCH_VERSION_SUFFIX="+$CU_VERSION" fi - if [[ "$(uname)" == Darwin ]]; then - export PYTORCH_VERSION_SUFFIX="" - else - export PYTORCH_VERSION_SUFFIX="+cpu" + # Match the suffix scheme of pytorch, unless this package does not have + # CUDA builds (in which case, use default) + if [[ -z "$NO_CUDA_PACKAGE" ]]; then + export VERSION_SUFFIX="$PYTORCH_VERSION_SUFFIX" + # If the suffix is non-empty, we will use a wheel subdirectory + if [[ -n "$PYTORCH_VERSION_SUFFIX" ]]; then + export WHEEL_DIR="$PYTORCH_VERSION_SUFFIX/" + fi fi - export VERSION_SUFFIX="" - # NB: When there is no CUDA package available, we put these - # packages in the top-level directory, so they are eligible - # for selection even if you are otherwise trying to install - # a cu100 stack. This differs from when there ARE CUDA packages - # available; then we don't want the cpu package; we want - # to give you as much goodies as possible. - export WHEEL_DIR="" - else - case "$CU_VERSION" in - cu100) - export PYTORCH_VERSION_SUFFIX="" - export CUDA_HOME=/usr/local/cuda-10.0/ - export FORCE_CUDA=1 - # Hard-coding gencode flags is temporary situation until - # https://github.com/pytorch/pytorch/pull/23408 lands - export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50" - ;; - cu92) - export CUDA_HOME=/usr/local/cuda-9.2/ - export PYTORCH_VERSION_SUFFIX="+cu92" - export FORCE_CUDA=1 - export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50" - ;; - cpu) - export PYTORCH_VERSION_SUFFIX="+cpu" - ;; - *) - echo "Unrecognized CU_VERSION=$CU_VERSION" - esac - export VERSION_SUFFIX="$PYTORCH_VERSION_SUFFIX" - export WHEEL_DIR="$CU_VERSION/" fi + + # Now work out the CUDA settings + case "$CU_VERSION" in + cu100) + export CUDA_HOME=/usr/local/cuda-10.0/ + export FORCE_CUDA=1 + # Hard-coding gencode flags is temporary situation until + # https://github.com/pytorch/pytorch/pull/23408 lands + export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50" + ;; + cu92) + export CUDA_HOME=/usr/local/cuda-9.2/ + export FORCE_CUDA=1 + export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50" + ;; + cpu) + ;; + *) + echo "Unrecognized CU_VERSION=$CU_VERSION" + exit 1 + ;; + esac } # Populate build version if necessary, and add version suffix # # Inputs: # BUILD_VERSION (e.g., 0.2.0 or empty) +# VERSION_SUFFIX (e.g., +cpu) # # Outputs: # BUILD_VERSION (e.g., 0.2.0.dev20190807+cpu) From 82d4cf1bcef086aec24785f3d2e49324a186729c Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Wed, 9 Oct 2019 13:00:03 -0700 Subject: [PATCH 12/17] 0.3.1 point release Signed-off-by: Edward Z. Yang --- .circleci/config.yml | 154 ++++++++++++++++++++++++++++++++++++++- .circleci/config.yml.in | 8 +- packaging/build_conda.sh | 2 +- packaging/build_wheel.sh | 2 +- 4 files changed, 156 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cc15b07ec1..239b5c557f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,11 +12,11 @@ binary_common: &binary_common build_version: description: "version number of release binary; by default, build a nightly" type: string - default: "0.3.0" + default: "0.3.1" pytorch_version: description: "PyTorch version to build against; by default, use a nightly" type: string - default: "1.2.0" + default: "1.3.0" # Don't edit these python_version: description: "Python version to build against (e.g., 3.7)" @@ -159,7 +159,7 @@ workflows: build: jobs: - circleci_consistency - + # - binary_linux_wheel: name: binary_linux_wheel_py2.7 python_version: "2.7" @@ -216,4 +216,150 @@ workflows: - binary_macos_conda: name: binary_macos_conda_py3.7 python_version: "3.7" - # workflows(prefix="nightly_", upload=True) \ No newline at end of file + + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py2.7 + python_version: "2.7" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py2.7_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py2.7 + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py2.7_unicode_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py2.7_unicode + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py3.5 + python_version: "3.5" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py3.5_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py3.5 + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py3.6 + python_version: "3.6" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py3.6_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py3.6 + - binary_linux_wheel: + name: nightly_binary_linux_wheel_py3.7 + python_version: "3.7" + - binary_wheel_upload: + name: nightly_binary_linux_wheel_py3.7_upload + context: org-member + requires: + - nightly_binary_linux_wheel_py3.7 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py2.7 + python_version: "2.7" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py2.7_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py2.7 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py2.7_unicode + python_version: "2.7" + unicode_abi: "1" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py2.7_unicode_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py2.7_unicode + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py3.5 + python_version: "3.5" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py3.5_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py3.5 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py3.6 + python_version: "3.6" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py3.6_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py3.6 + - binary_macos_wheel: + name: nightly_binary_macos_wheel_py3.7 + python_version: "3.7" + - binary_wheel_upload: + name: nightly_binary_macos_wheel_py3.7_upload + context: org-member + requires: + - nightly_binary_macos_wheel_py3.7 + - binary_linux_conda: + name: nightly_binary_linux_conda_py2.7 + python_version: "2.7" + - binary_conda_upload: + name: nightly_binary_linux_conda_py2.7_upload + context: org-member + requires: + - nightly_binary_linux_conda_py2.7 + - binary_linux_conda: + name: nightly_binary_linux_conda_py3.5 + python_version: "3.5" + - binary_conda_upload: + name: nightly_binary_linux_conda_py3.5_upload + context: org-member + requires: + - nightly_binary_linux_conda_py3.5 + - binary_linux_conda: + name: nightly_binary_linux_conda_py3.6 + python_version: "3.6" + - binary_conda_upload: + name: nightly_binary_linux_conda_py3.6_upload + context: org-member + requires: + - nightly_binary_linux_conda_py3.6 + - binary_linux_conda: + name: nightly_binary_linux_conda_py3.7 + python_version: "3.7" + - binary_conda_upload: + name: nightly_binary_linux_conda_py3.7_upload + context: org-member + requires: + - nightly_binary_linux_conda_py3.7 + - binary_macos_conda: + name: nightly_binary_macos_conda_py2.7 + python_version: "2.7" + - binary_conda_upload: + name: nightly_binary_macos_conda_py2.7_upload + context: org-member + requires: + - nightly_binary_macos_conda_py2.7 + - binary_macos_conda: + name: nightly_binary_macos_conda_py3.5 + python_version: "3.5" + - binary_conda_upload: + name: nightly_binary_macos_conda_py3.5_upload + context: org-member + requires: + - nightly_binary_macos_conda_py3.5 + - binary_macos_conda: + name: nightly_binary_macos_conda_py3.6 + python_version: "3.6" + - binary_conda_upload: + name: nightly_binary_macos_conda_py3.6_upload + context: org-member + requires: + - nightly_binary_macos_conda_py3.6 + - binary_macos_conda: + name: nightly_binary_macos_conda_py3.7 + python_version: "3.7" + - binary_conda_upload: + name: nightly_binary_macos_conda_py3.7_upload + context: org-member + requires: + - nightly_binary_macos_conda_py3.7 \ No newline at end of file diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index db1fa083d7..2cdea55090 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -12,11 +12,11 @@ binary_common: &binary_common build_version: description: "version number of release binary; by default, build a nightly" type: string - default: "0.3.0" + default: "0.3.1" pytorch_version: description: "PyTorch version to build against; by default, use a nightly" type: string - default: "1.2.0" + default: "1.3.0" # Don't edit these python_version: description: "Python version to build against (e.g., 3.7)" @@ -189,5 +189,5 @@ workflows: build: jobs: - circleci_consistency - {{ workflows() }} - # workflows(prefix="nightly_", upload=True) + # {{ workflows() }} + {{ workflows(prefix="nightly_", upload=True) }} diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index 47b428e773..bb828929f5 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -6,7 +6,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export BUILD_TYPE="conda" export NO_CUDA_PACKAGE=1 -setup_env 0.4.0 +setup_env 0.3.1 export SOURCE_ROOT_DIR="$PWD" setup_conda_pytorch_constraint conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchaudio diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 00def0928e..c7332519cf 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -6,7 +6,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export BUILD_TYPE="wheel" export NO_CUDA_PACKAGE=1 -setup_env 0.4.0 +setup_env 0.3.1 setup_wheel_python "$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies pip_install numpy future From 257c18c90746cb316238bbd10b140a873111018c Mon Sep 17 00:00:00 2001 From: engineerchuan Date: Mon, 16 Sep 2019 15:26:51 -0400 Subject: [PATCH 13/17] Add a retry and change timeout to adjust for network failures when downloading from sourceforge (#280) * Add a retry to adjust for network failures when downloading from sourceforge * try to fix packaging by adding longer connection timeout --- packaging/build_from_source.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packaging/build_from_source.sh b/packaging/build_from_source.sh index 3e16da5ede..082353a2b6 100755 --- a/packaging/build_from_source.sh +++ b/packaging/build_from_source.sh @@ -10,10 +10,12 @@ rm -rf /tmp/torchaudio-deps mkdir /tmp/torchaudio-deps pushd /tmp/torchaudio-deps -curl -L -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" -curl -L -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" -curl -L -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" -curl -L -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" +CURL_OPTS=" --retry 8 --connect-timeout 8 " + +curl -L $CURL_OPTS -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" +curl -L $CURL_OPTS -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" +curl -L $CURL_OPTS -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" +curl -L $CURL_OPTS -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" # unpack the dependencies tar xfp sox-14.4.2.tar.bz2 From a263118ab85c86288d092a1171281d4187bce93e Mon Sep 17 00:00:00 2001 From: engineerchuan Date: Wed, 18 Sep 2019 11:03:23 -0400 Subject: [PATCH 14/17] Curl Settings for Build Script (#284) * Add a retry to adjust for network failures when downloading from sourceforge * fix packaging by adding longer connection timeout --- packaging/build_from_source.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packaging/build_from_source.sh b/packaging/build_from_source.sh index 082353a2b6..240cc4600d 100755 --- a/packaging/build_from_source.sh +++ b/packaging/build_from_source.sh @@ -10,12 +10,16 @@ rm -rf /tmp/torchaudio-deps mkdir /tmp/torchaudio-deps pushd /tmp/torchaudio-deps -CURL_OPTS=" --retry 8 --connect-timeout 8 " -curl -L $CURL_OPTS -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" -curl -L $CURL_OPTS -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" -curl -L $CURL_OPTS -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" -curl -L $CURL_OPTS -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" +# Curl Settings +# 3 minutes is the absolute max for the curl command +# Retry up to 10 times, wait to connect at most 5s per time +CURL_OPTS="-L --retry 10 --connect-timeout 5 --max-time 180" + +curl $CURL_OPTS -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2" +curl $CURL_OPTS -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz" +curl $CURL_OPTS -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" +curl $CURL_OPTS -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" # unpack the dependencies tar xfp sox-14.4.2.tar.bz2 From 120f69311050d3b96cc89b5c0f562dc63731f345 Mon Sep 17 00:00:00 2001 From: engineerchuan Date: Wed, 25 Sep 2019 14:35:45 -0400 Subject: [PATCH 15/17] Updated URL for downloading flac, changed http to https download (#297) * Updated URL for downloading flac, changed http to https download * Removed comments that are documented in the code --- packaging/build_from_source.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packaging/build_from_source.sh b/packaging/build_from_source.sh index 240cc4600d..6bdad82c5c 100755 --- a/packaging/build_from_source.sh +++ b/packaging/build_from_source.sh @@ -12,13 +12,11 @@ pushd /tmp/torchaudio-deps # Curl Settings -# 3 minutes is the absolute max for the curl command -# Retry up to 10 times, wait to connect at most 5s per time CURL_OPTS="-L --retry 10 --connect-timeout 5 --max-time 180" -curl $CURL_OPTS -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2" -curl $CURL_OPTS -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz" -curl $CURL_OPTS -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" +curl $CURL_OPTS -o sox-14.4.2.tar.bz2 "https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2" +curl $CURL_OPTS -o lame-3.99.5.tar.gz "https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz" +curl $CURL_OPTS -o flac-1.3.2.tar.xz "https://downloads.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" curl $CURL_OPTS -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" # unpack the dependencies From ecffe24db5b59101d2301d0e7b11a87189731483 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Fri, 11 Oct 2019 10:24:48 -0700 Subject: [PATCH 16/17] disable uploads Signed-off-by: Edward Z. Yang --- .circleci/config.yml.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 2cdea55090..9ce46bc4b5 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -189,5 +189,5 @@ workflows: build: jobs: - circleci_consistency - # {{ workflows() }} - {{ workflows(prefix="nightly_", upload=True) }} + {{ workflows() }} + # {{ workflows(prefix="nightly_", upload=True) }} From be38af3f624a91f39280f75c7a354429eaf423c2 Mon Sep 17 00:00:00 2001 From: Vincent Quenneville-Belair Date: Mon, 25 Nov 2019 16:36:56 -0500 Subject: [PATCH 17/17] 0.3.2 point release. --- .circleci/config.yml | 4 ++-- .circleci/config.yml.in | 4 ++-- packaging/build_conda.sh | 2 +- packaging/build_wheel.sh | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 239b5c557f..52cdda70b8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,11 +12,11 @@ binary_common: &binary_common build_version: description: "version number of release binary; by default, build a nightly" type: string - default: "0.3.1" + default: "0.3.2" pytorch_version: description: "PyTorch version to build against; by default, use a nightly" type: string - default: "1.3.0" + default: "1.3.1" # Don't edit these python_version: description: "Python version to build against (e.g., 3.7)" diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 9ce46bc4b5..e0b5378d73 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -12,11 +12,11 @@ binary_common: &binary_common build_version: description: "version number of release binary; by default, build a nightly" type: string - default: "0.3.1" + default: "0.3.2" pytorch_version: description: "PyTorch version to build against; by default, use a nightly" type: string - default: "1.3.0" + default: "1.3.1" # Don't edit these python_version: description: "Python version to build against (e.g., 3.7)" diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index bb828929f5..2dbf064e03 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -6,7 +6,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export BUILD_TYPE="conda" export NO_CUDA_PACKAGE=1 -setup_env 0.3.1 +setup_env 0.3.2 export SOURCE_ROOT_DIR="$PWD" setup_conda_pytorch_constraint conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchaudio diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index c7332519cf..84b844eb2e 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -6,7 +6,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export BUILD_TYPE="wheel" export NO_CUDA_PACKAGE=1 -setup_env 0.3.1 +setup_env 0.3.2 setup_wheel_python "$script_dir/build_from_source.sh" "$(pwd)" # Build static dependencies pip_install numpy future