From 724cef38007977711a6355587c7d22976e0470da Mon Sep 17 00:00:00 2001 From: Michael Joseph Date: Wed, 23 Oct 2013 09:21:59 +0200 Subject: [PATCH 01/12] Use subshell to avoid changing directories --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d16f2eb..ef18791 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ Lint the project with: Generate the documentation with: - cd docs && PYTHONPATH=.. make singlehtml + (cd docs && make singlehtml) To monitor changes to Python files and execute flake8 and nosetests automatically, execute the following from the root project directory: From 701d32007916ff6a1278894db0eed46e6d4a6a56 Mon Sep 17 00:00:00 2001 From: Michael Joseph Date: Wed, 23 Oct 2013 12:56:33 +0200 Subject: [PATCH 02/12] Wrap sh calls to check for dry_run --- changes/shell.py | 14 +++++--------- tests/test_shell.py | 25 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 9 deletions(-) create mode 100644 tests/test_shell.py diff --git a/changes/shell.py b/changes/shell.py index fec3908..fff2f9e 100644 --- a/changes/shell.py +++ b/changes/shell.py @@ -1,17 +1,13 @@ import logging -import iterpipes +from changes import config log = logging.getLogger(__name__) -def execute(command, dry_run=True): - log.debug('executing %s', command) - if not dry_run: - try: - return [result for result in iterpipes.linecmd(command)(None)] - except iterpipes.CalledProcessError, e: - log.debug('return code: %s, output: %s', e.returncode, e.output) - return False +def handle_dry_run(function, *args): + if not config.arguments.get('--dry-run', True): + return function(*args) else: + log.debug('dry run of %s %s, skipping' % (function, args)) return True diff --git a/tests/test_shell.py b/tests/test_shell.py new file mode 100644 index 0000000..30c9a09 --- /dev/null +++ b/tests/test_shell.py @@ -0,0 +1,25 @@ +import sh +from unittest2 import TestCase + +from changes import config, shell + + +class ShellTestCase(TestCase): + + def test_handle_dry_run(self): + self.assertEquals( + '', + shell.handle_dry_run( + sh.diff, + ('README.md', 'README.md') + ) + ) + + def test_handle_dry_run_true(self): + config.arguments['--dry-run'] = True + self.assertTrue( + shell.handle_dry_run( + sh.diff, + ('README.md', 'README.md') + ) + ) From 3509b84ae6bd179696699271ed5d2c6f89e1e2e8 Mon Sep 17 00:00:00 2001 From: Michael Joseph Date: Thu, 24 Oct 2013 10:26:25 +0200 Subject: [PATCH 03/12] Rename testing to verification Don't make nosetests run itself --- changes/cli.py | 4 ++-- changes/testing.py | 24 ------------------------ changes/verification.py | 27 +++++++++++++++++++++++++++ tests/test_testing.py | 11 ----------- 4 files changed, 29 insertions(+), 37 deletions(-) delete mode 100644 changes/testing.py create mode 100644 changes/verification.py delete mode 100644 tests/test_testing.py diff --git a/changes/cli.py b/changes/cli.py index f8234bc..e5696f8 100644 --- a/changes/cli.py +++ b/changes/cli.py @@ -49,7 +49,7 @@ from changes.config import arguments from changes.changelog import changelog from changes.packaging import install, upload, pypi -from changes.testing import run_tests +from changes.verification import run_tests from changes.version import bump_version from changes.vcs import tag, commit_version_change @@ -61,7 +61,7 @@ def release(): if not arguments['--skip-changelog']: changelog() bump_version() - test() + run_tests() commit_version_change() install() upload() diff --git a/changes/testing.py b/changes/testing.py deleted file mode 100644 index e490735..0000000 --- a/changes/testing.py +++ /dev/null @@ -1,24 +0,0 @@ -import tempfile -import logging - -from changes import shell - -log = logging.getLogger(__name__) - - -def run_tests(): - command = 'nosetests' - if arguments['--tox']: - command = 'tox' - - if not shell.execute(command, dry_run=False): - raise Exception('Test command failed') - - -def run_test_command(): - if arguments['--test-command']: - test_command = arguments['--test-command'] - result = shell.execute(test_command, dry_run=arguments['--dry-run']) - log.info('Test command "%s", returned %s', test_command, result) - else: - log.warning('Test command "%s" failed', test_command) diff --git a/changes/verification.py b/changes/verification.py new file mode 100644 index 0000000..6e9f67f --- /dev/null +++ b/changes/verification.py @@ -0,0 +1,27 @@ +import logging + +import sh + +from changes import config, shell + +log = logging.getLogger(__name__) + + +def run_tests(): + if config.arguments['--tox']: + result = sh.tox() + else: + result = sh.nosetests() + + if not result: + raise Exception('Test command failed') + else: + return True + + +def run_test_command(): + if config.arguments['--test-command']: + test_command = config.arguments['--test-command'] + result = shell.execute(sh, tuple(test_command.split(' '))) + log.info('Test command "%s", returned %s', test_command, result) + return True diff --git a/tests/test_testing.py b/tests/test_testing.py deleted file mode 100644 index b4fb99a..0000000 --- a/tests/test_testing.py +++ /dev/null @@ -1,11 +0,0 @@ -from changes import config, testing -from . import BaseTestCase - - -class TestingTestCase(BaseTestCase): - """ - def test(): - def make_virtualenv(): - def run_test_command(): - """ - pass \ No newline at end of file From 2c202e1fb19dfacc5a20df46678628daf0eb5064 Mon Sep 17 00:00:00 2001 From: Michael Joseph Date: Thu, 24 Oct 2013 10:27:12 +0200 Subject: [PATCH 04/12] Port attributes to sh --- changes/attributes.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/changes/attributes.py b/changes/attributes.py index 87c1c66..ee2e352 100644 --- a/changes/attributes.py +++ b/changes/attributes.py @@ -4,7 +4,7 @@ from path import path -from changes import shell +import sh log = logging.getLogger(__name__) @@ -32,10 +32,7 @@ def replace_attribute(app_name, attribute_name, new_value, dry_run=True): if not dry_run: path(tmp_file).move(init_file) else: - log.debug(shell.execute( - 'diff %s %s' % (tmp_file, init_file), - dry_run=False - )) + log.debug(sh.diff(tmp_file, init_file, _ok_code=1)) def has_attribute(app_name, attribute_name): From eabc7018e3a5f0d96e2d40fdd5592af3e57015d5 Mon Sep 17 00:00:00 2001 From: Michael Joseph Date: Thu, 24 Oct 2013 10:27:27 +0200 Subject: [PATCH 05/12] Port changelog to sh --- changes/changelog.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/changes/changelog.py b/changes/changelog.py index 993dd07..9b96567 100644 --- a/changes/changelog.py +++ b/changes/changelog.py @@ -1,7 +1,8 @@ import logging import re -from changes import attributes, config, shell, version +import sh +from changes import attributes, config, version log = logging.getLogger(__name__) @@ -64,21 +65,21 @@ def changelog(): ) ] - ## vcs (todo: config templatise all these commands) - git_log = 'git log --oneline --no-merges' - version_difference = '%s..master' % version.current_version(app_name) - - git_log_content = shell.execute( - '%s %s' % (git_log, version_difference), - dry_run=False - ) + git_log_content = sh.git.log( + '--oneline', + '--no-merges', + '%s..master' % version.current_version(app_name), + _tty_out=False + ).split('\n') log.debug('content: %s' % git_log_content) if not git_log_content: - log.debug('sniffing initial release, drop tags: %s', git_log) - git_log_content = shell.execute(git_log, dry_run=False) - - ## /vcs + log.debug('sniffing initial release, drop tags') + git_log_content = sh.git.log( + '--oneline', + '--no-merges', + _tty_out=False + ).split('\n') git_log_content = replace_sha_with_commit_link(git_log_content) @@ -87,10 +88,9 @@ def changelog(): # makes change log entries into bullet points if git_log_content: [ - changelog_content.append('* %s' % line) + changelog_content.append('* %s\n' % line) if line else line for line in git_log_content[:-1] - # for all except the last line? ] write_new_changelog( From f3fc756a06e7150ab95d6622e01647f349de5a66 Mon Sep 17 00:00:00 2001 From: Michael Joseph Date: Thu, 24 Oct 2013 10:30:43 +0200 Subject: [PATCH 06/12] iterpipes => sh --- requirements.txt | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7473950..202be53 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ coverage < 4.0.0 docopt < 1.0.0 -iterpipes < 1.0.0 flake8 < 3.0.0 mock < 2.0.0 nose < 2.0.0 @@ -9,6 +8,7 @@ path.py < 5.0.0 pinocchio < 1.0.0 python-coveralls < 3.0.0 semantic_version < 3.0.0 +sh < 2.0.0 sphinx-bootstrap-theme < 1.0.0 sphinxcontrib-httpdomain < 2.0.0 testtube < 1.0.0 diff --git a/setup.py b/setup.py index 0c9a2b2..7a83f3c 100644 --- a/setup.py +++ b/setup.py @@ -18,10 +18,10 @@ packages=['changes'], install_requires=[ 'docopt < 1.0.0', - 'iterpipes < 1.0.0', - 'pypandoc < 1.0.0', 'path.py < 5.0.0', + 'pypandoc < 1.0.0', 'semantic_version < 3.0.0', + 'sh < 2.0.0', 'virtualenv < 2.0.0', ], entry_points={ From 8667bd836adc03aa71bbd7bebcfd276dc3680f7a Mon Sep 17 00:00:00 2001 From: Michael Joseph Date: Thu, 24 Oct 2013 10:36:59 +0200 Subject: [PATCH 07/12] Migrate vcs to sh --- changes/vcs.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/changes/vcs.py b/changes/vcs.py index fab18fe..4465eb6 100644 --- a/changes/vcs.py +++ b/changes/vcs.py @@ -1,22 +1,35 @@ +import logging + +import sh + from changes import config, shell +log = logging.getLogger(__name__) + def commit_version_change(): app_name, dry_run, new_version = config.common_arguments() - command = 'git commit -m %s %s/__init__.py %s' % ( - new_version, app_name, config.CHANGELOG + commit_result = shell.handle_dry_run( + sh.git.commit, + ('-m', new_version, '%s/__init__.py' % app_name, config.CHANGELOG) ) - if not (shell.execute(command, dry_run=dry_run) and - shell.execute('git push', dry_run=dry_run)): - raise Exception('Version change commit failed') + if commit_result: + push_result = shell.handle_dry_run(sh.git.push, ()) + if not push_result: + raise Exception('Version change commit failed') + def tag(): _, dry_run, new_version = config.common_arguments() - shell.execute( - 'git tag -a %s -m "%s"' % (new_version, new_version), - dry_run=dry_run + tag_result = shell.handle_dry_run( + sh.git.tag, + ('-a', new_version, '-m', '"%s"' % new_version) ) - shell.execute('git push --tags', dry_run=dry_run) + + if tag_result: + push_tags_result = shell.handle_dry_run(sh.git.push, ('--tags')) + if not push_tags_result: + raise Exception('Tagging failed') From 4fce9d27d1c16bfd94353b1632db3b6f70866802 Mon Sep 17 00:00:00 2001 From: Michael Joseph Date: Thu, 24 Oct 2013 10:37:22 +0200 Subject: [PATCH 08/12] Migrate probe to sh --- changes/probe.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/changes/probe.py b/changes/probe.py index 1685227..3a40dd4 100644 --- a/changes/probe.py +++ b/changes/probe.py @@ -1,7 +1,9 @@ import logging from os.path import exists -from changes import attributes, shell +import sh + +from changes import attributes log = logging.getLogger(__name__) @@ -18,7 +20,7 @@ def probe_project(app_name): """ log.info('Checking project for changes requirements.') # on [github](https://github.com) - git_remotes = shell.execute('git remote -v', dry_run=False) + git_remotes = sh.git.remote('-v') on_github = any(['github.com' in remote for remote in git_remotes]) log.info('On Github? %s', on_github) @@ -45,7 +47,6 @@ def probe_project(app_name): log.info('Has module metadata? %s', has_metadata) # supports executing tests with `nosetests` or `tox` - log.debug(requirements_contents) runs_tests = ( has_requirement('nose', requirements_contents) or has_requirement('tox', requirements_contents) From 61a59575ba88921538f9176724e6101ac54fa7e3 Mon Sep 17 00:00:00 2001 From: Michael Joseph Date: Thu, 24 Oct 2013 10:39:10 +0200 Subject: [PATCH 09/12] Fix versioning --- changes/cli.py | 6 ++++-- changes/version.py | 6 ++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/changes/cli.py b/changes/cli.py index e5696f8..3826e95 100644 --- a/changes/cli.py +++ b/changes/cli.py @@ -45,7 +45,7 @@ from docopt import docopt import changes -from changes import probe +from changes import config, probe, util, version from changes.config import arguments from changes.changelog import changelog from changes.packaging import install, upload, pypi @@ -83,9 +83,11 @@ def initialise(): def main(): arguments = initialise() + version_arguments = ['--major', '--minor', '--patch'] commands = ['release', 'changelog', 'run_tests', 'bump_version', 'tag', 'upload', 'install', 'pypi'] suppress_version_prompt_for = ['run_tests', 'upload'] + if arguments['--new-version']: arguments['new_version'] = arguments['--new-version'] @@ -100,6 +102,6 @@ def main(): arguments['new_version'] = version.get_new_version( app_name, version.current_version(app_name), - **version.extract_version_arguments(arguments) + **util.extract_arguments(arguments, version_arguments) ) globals()[command]() diff --git a/changes/version.py b/changes/version.py index 9a1106e..28bbcbe 100644 --- a/changes/version.py +++ b/changes/version.py @@ -2,7 +2,7 @@ import semantic_version -from changes import config, util, attributes +from changes import config, attributes log = logging.getLogger(__name__) @@ -35,17 +35,15 @@ def get_new_version(app_name, current_version, 'What is the release version for "%s" ' '[Default: %s]: ' % ( app_name, proposed_new_version - **util.extract_arguments( ) ) + if not new_version: return proposed_new_version.strip() else: return new_version.strip() - - def increment(version, major=False, minor=False, patch=True): """ Increment a semantic version From b2a6c8c379c83f71325d91bad2b09edda37edf9a Mon Sep 17 00:00:00 2001 From: Michael Joseph Date: Thu, 24 Oct 2013 10:39:36 +0200 Subject: [PATCH 10/12] Port packaging to sh --- changes/packaging.py | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/changes/packaging.py b/changes/packaging.py index faff806..e535607 100644 --- a/changes/packaging.py +++ b/changes/packaging.py @@ -1,7 +1,13 @@ -import virtualenv +import logging +import tempfile + from path import path +import sh +import virtualenv -from changes import config, shell, testing +from changes import config, shell, verification + +log = logging.getLogger(__name__) def make_virtualenv(): @@ -13,19 +19,19 @@ def make_virtualenv(): def install(): app_name, dry_run, new_version = config.common_arguments() - result = shell.execute('python setup.py clean sdist', dry_run=dry_run) + result = shell.handle_dry_run(sh.python, ('setup.py', 'clean', 'sdist')) if result: tmp_dir = make_virtualenv() try: virtualenv.install_sdist( - arguments[''], + config.arguments[''], 'dist/%s-%s.tar.gz' % (app_name, new_version), '%s/bin/python' % tmp_dir ) log.info('Successfully installed %s sdist', app_name) - if testing.run_test_command(): + if verification.run_test_command(): log.info('Successfully ran test command: %s', - arguments['--test-command']) + config.arguments['--test-command']) except: raise Exception('Error installing %s sdist', app_name) @@ -34,13 +40,14 @@ def install(): def upload(): app_name, dry_run, new_version = config.common_arguments() - pypi = arguments['--pypi'] + pypi = config.arguments['--pypi'] - upload = 'python setup.py clean sdist upload' + upload_args = 'setup.py clean sdist upload'.split(' ') if pypi: - upload = upload + '-r %s' % pypi + upload_args.extend(['-r', pypi]) - if not shell.execute(upload, dry_run=dry_run): + upload_result = shell.handle_dry_run(sh.python, tuple(upload_args)) + if not upload_result: raise Exception('Error uploading') else: log.info('Succesfully uploaded %s %s', app_name, new_version) @@ -48,18 +55,20 @@ def upload(): def pypi(): app_name, dry_run, _ = config.common_arguments() - pypi = arguments['--pypi'] - package_index = 'pypi' tmp_dir = make_virtualenv() - install = '%s/bin/pip install %s' % (tmp_dir, app_name) + install_args = ('%s/bin/pip install %s' % (tmp_dir, app_name)).split(' ') + package_index = 'pypi' + pypi = config.arguments['--pypi'] if pypi: - install = install + '-i %s' % pypi + install_args.extend('-i', pypi) package_index = pypi + install_args = tuple(install_args) + log.debug(install_args) try: - result = shell.execute(install, dry_run=dry_run) + result = shell.handle_dry_run(sh, install_args) if result: log.info('Successfully installed %s from %s', app_name, package_index) @@ -67,8 +76,9 @@ def pypi(): log.error('Failed to install %s from %s', app_name, package_index) - testing.run_test_command() + verification.run_test_command() except: + log.exception('error installing %s from %s', app_name, package_index) raise Exception('Error installing %s from %s', app_name, package_index) path(tmp_dir).rmtree(path(tmp_dir)) From 77d540f9575e2aceb4482dafe461dfb906465f6e Mon Sep 17 00:00:00 2001 From: Michael Joseph Date: Thu, 24 Oct 2013 11:02:07 +0200 Subject: [PATCH 11/12] Add subprocess execution for testing installation in a virtualenv --- changes/compat.py | 23 +++++++++++++++++++++++ changes/packaging.py | 8 +++----- changes/shell.py | 13 +++++++++++++ tests/test_packaging.py | 19 +++++++++++++------ 4 files changed, 52 insertions(+), 11 deletions(-) create mode 100644 changes/compat.py diff --git a/changes/compat.py b/changes/compat.py new file mode 100644 index 0000000..b161f39 --- /dev/null +++ b/changes/compat.py @@ -0,0 +1,23 @@ +from subprocess import Popen, PIPE, CalledProcessError + + +def check_output(*popenargs, **kwargs): + """ + Run command with arguments and return its output as a byte string. + + Backported from Python 2.7 as it's implemented as pure python on stdlib. + + >>> check_output(['/usr/bin/python', '--version']) + Python 2.6.2 + """ + process = Popen(stdout=PIPE, *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + error = CalledProcessError(retcode, cmd) + error.output = output + raise error + return output diff --git a/changes/packaging.py b/changes/packaging.py index e535607..74e4ef8 100644 --- a/changes/packaging.py +++ b/changes/packaging.py @@ -57,18 +57,16 @@ def pypi(): app_name, dry_run, _ = config.common_arguments() tmp_dir = make_virtualenv() - install_args = ('%s/bin/pip install %s' % (tmp_dir, app_name)).split(' ') + install_cmd = '%s/bin/pip install %s' % (tmp_dir, app_name) package_index = 'pypi' pypi = config.arguments['--pypi'] if pypi: - install_args.extend('-i', pypi) + install_cmd += '-i %s' % pypi package_index = pypi - install_args = tuple(install_args) - log.debug(install_args) try: - result = shell.handle_dry_run(sh, install_args) + result = shell.execute(install_cmd, dry_run=dry_run) if result: log.info('Successfully installed %s from %s', app_name, package_index) diff --git a/changes/shell.py b/changes/shell.py index fff2f9e..acd1412 100644 --- a/changes/shell.py +++ b/changes/shell.py @@ -1,6 +1,7 @@ import logging from changes import config +from changes.compat import check_output, CalledProcessError log = logging.getLogger(__name__) @@ -11,3 +12,15 @@ def handle_dry_run(function, *args): else: log.debug('dry run of %s %s, skipping' % (function, args)) return True + + +def execute(command, dry_run=True): + if not dry_run: + try: + return check_output(command.split(' ')).split('\n') + except CalledProcessError as e: + log.debug('return code: %s, output: %s', e.returncode, e.output) + return False + else: + log.debug('dry run of %s, skipping' % command) + return True diff --git a/tests/test_packaging.py b/tests/test_packaging.py index f318b32..1699bc9 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -1,10 +1,17 @@ -from changes import config, packaging +from changes import packaging from . import BaseTestCase class PackagingTestCase(BaseTestCase): - # def make_virtualenv(): - # def install(): - # def upload(): - # def pypi(): - pass \ No newline at end of file + + def test_install(self): + packaging.install() + + def test_make_virtualenv(self): + packaging.make_virtualenv() + + def test_upload(self): + packaging.upload() + + def test_pypi(self): + packaging.pypi() From bb935fa4f5242e4a4c4de5554e1143fc3ed0f168 Mon Sep 17 00:00:00 2001 From: Michael Joseph Date: Thu, 24 Oct 2013 12:00:40 +0200 Subject: [PATCH 12/12] Dry runs --- tests/test_packaging.py | 5 ++++- tests/test_shell.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 1699bc9..91d1bca 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -1,17 +1,20 @@ -from changes import packaging +from changes import config, packaging from . import BaseTestCase class PackagingTestCase(BaseTestCase): def test_install(self): + config.arguments['--dry-run'] = True packaging.install() def test_make_virtualenv(self): packaging.make_virtualenv() def test_upload(self): + config.arguments['--dry-run'] = True packaging.upload() def test_pypi(self): + config.arguments['--dry-run'] = True packaging.pypi() diff --git a/tests/test_shell.py b/tests/test_shell.py index 30c9a09..a952cbb 100644 --- a/tests/test_shell.py +++ b/tests/test_shell.py @@ -7,6 +7,7 @@ class ShellTestCase(TestCase): def test_handle_dry_run(self): + config.arguments['--dry-run'] = False self.assertEquals( '', shell.handle_dry_run(