From 0db48dfd46d741bcd1cece6ad4424ba81e70bc10 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Thu, 29 Oct 2015 21:16:38 +0100 Subject: [PATCH 01/44] Add experimental php features for pydiploy --- pydiploy/examples/django_fabfile.py | 2 + pydiploy/php.py | 116 +++++++++ pydiploy/require/__init__.py | 2 +- pydiploy/require/apache.py | 249 +++++++++++++++++++ pydiploy/templates/apache_host.conf.tpl | 18 ++ pydiploy/templates/apache_host_down.conf.tpl | 14 ++ tools/pydiployfabfile.sublime-snippet | 11 +- 7 files changed, 409 insertions(+), 3 deletions(-) create mode 100644 pydiploy/php.py create mode 100644 pydiploy/require/apache.py create mode 100644 pydiploy/templates/apache_host.conf.tpl create mode 100644 pydiploy/templates/apache_host_down.conf.tpl diff --git a/pydiploy/examples/django_fabfile.py b/pydiploy/examples/django_fabfile.py index ea4110b..ecfe79d 100644 --- a/pydiploy/examples/django_fabfile.py +++ b/pydiploy/examples/django_fabfile.py @@ -104,6 +104,7 @@ def test(): env.short_server_name = 'myapp-dev' env.static_folder = '/site_media/' env.server_ip = '192.168.1.3' + #env.port = '1664' env.no_shared_sessions = False env.server_ssl_on = False env.goal = 'test' @@ -128,6 +129,7 @@ def prod(): env.short_server_name = 'myapp' env.static_folder = '/site_media/' env.server_ip = '' + #env.port = '' env.no_shared_sessions = False env.server_ssl_on = True env.path_to_cert = '/etc/ssl/certs/myapp.net.pem' diff --git a/pydiploy/php.py b/pydiploy/php.py new file mode 100644 index 0000000..6a053ba --- /dev/null +++ b/pydiploy/php.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- + +""" This module is used to deploy a whole php webapp using chaussette/circus nginx on a remote/vagrant machine. + +This module shoud be imported in a fabfile to deploy an application using pydiploy. + +""" + + +from contextlib import contextmanager + +import fabric +import fabtools +import pydiploy +from fabric.api import env +from pydiploy.decorators import do_verbose + + +@contextmanager +def wrap_deploy(): + try: + yield + except SystemExit: + fabric.api.execute(rollback) + fabric.api.abort(fabric.colors.red( + "Deploy failed rollbacking process launched")) + + +@do_verbose +def application_packages(update=False): + """ Installs all packages for php webapp """ + if env.has_key('extra_ppa_to_install'): + fabric.api.execute( + pydiploy.require.system.install_extra_ppa, env.extra_ppa_to_install) + if env.has_key('extra_pkg_to_install'): + fabric.api.execute( + pydiploy.require.system.install_extra_packages, env.extra_pkg_to_install) + + +def pre_install_backend(commands='/usr/bin/rsync'): + """ Installs requirements for apache / php """ + fabric.api.execute(pydiploy.require.system.add_user, commands=commands) + fabric.api.execute(pydiploy.require.system.set_locale) + fabric.api.execute(pydiploy.require.system.set_timezone) + fabric.api.execute(pydiploy.require.system.update_pkg_index) + fabric.api.execute(pydiploy.require.apache.apache_pkg) + fabtools.require.deb.packages(['php5','libapache2-mod-php5'], update=True) + fabric.api.execute(application_packages) + + +def deploy_backend(upgrade_pkg=False, **kwargs): + """ Deploys php webapp with required tag """ + with wrap_deploy(): + fabric.api.execute(pydiploy.require.releases_manager.setup) + fabric.api.execute(pydiploy.require.releases_manager.deploy_code) + fabric.api.execute(pydiploy.require.system.permissions) + fabric.api.execute(pydiploy.require.releases_manager.cleanup) + +def post_install_backend(): + fabric.api.execute(pydiploy.require.apache.web_configuration) + fabric.api.execute(pydiploy.require.apache.apache_restart) + + +def rollback(): + """ Rolls back php webapp """ + fabric.api.execute(pydiploy.require.releases_manager.rollback_code) + + +def reload_backend(): + """ Reloads backend """ + fabric.api.execute(pydiploy.require.apache.apache_reload) + + +def set_app_down(): + """ Sets app in maintenance mode """ + fabric.api.execute(pydiploy.require.apache.down_site_config) + fabric.api.execute(pydiploy.require.apache.set_website_down) + + +def set_app_up(): + """ Sets app in maintenance mode """ + fabric.api.execute(pydiploy.require.apache.set_website_up) + + +def install_postgres_server(user=None,dbname=None,password=None): + """ Install postgres server & add user for postgres + + if no parameters are provided using (if exists) :: + + default_db_user + default_db_name + default_db_password + + """ + + if not (user and dbname and password): + if all([e in env.keys() for e in ('default_db_user', 'default_db_name', 'default_db_password')]): + user = env.default_db_user + dbname = env.default_db_name + password = env.default_db_password + else: + fabric.api.abort('Please provide user,dbname,password parameters for postgres.') + + fabric.api.execute(pydiploy.require.databases.postgres.install_postgres_server) + fabric.api.execute(pydiploy.require.databases.postgres.add_postgres_user,user,password=password) + fabric.api.execute(pydiploy.require.databases.postgres.add_postgres_database,dbname,owner=user,locale=env.locale) + + +def install_oracle_client(): + """ Install oracle client. """ + fabric.api.execute(pydiploy.require.databases.oracle.install_oracle_client) + + +def install_sap_client(): + """ Install saprfc bindings to an SAP instance. """ + fabric.api.execute(pydiploy.require.databases.sap.install_sap_client) diff --git a/pydiploy/require/__init__.py b/pydiploy/require/__init__.py index 2e03b29..5285614 100644 --- a/pydiploy/require/__init__.py +++ b/pydiploy/require/__init__.py @@ -1,4 +1,4 @@ # -*- coding: utf-8 -*- -from pydiploy.require import (circus, databases, django, git, nginx, python, +from pydiploy.require import (apache, circus, databases, django, git, nginx, python, releases_manager, system) diff --git a/pydiploy/require/apache.py b/pydiploy/require/apache.py new file mode 100644 index 0000000..33101e7 --- /dev/null +++ b/pydiploy/require/apache.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- + +""" + +Apache webserver relatives methods +================================== + +""" +import os + +import fabric +import fabtools +import pydiploy +from fabric.api import env +from pydiploy.decorators import do_verbose + + +@do_verbose +def root_web(): + """ Creates web root for webserver """ + + fabtools.require.files.directory(env.remote_static_root, use_sudo=True, + owner='root', group='root', mode='755') + + +@do_verbose +def apache_pkg(update=False): + """ Installs apache package on remote server """ + + fabtools.require.deb.packages(['apache2'], update=update) + + +@do_verbose +def apache_start(): + """ Starts apache """ + + try: + if env.apache_start_confirmation: + fabric.api.execute(apache_confirm_start) + except: + fabtools.service.start('apache2') + + +@do_verbose +def apache_reload(): + """ Starts/reloads apache """ + + if not apache_started(): + fabric.api.execute(apache_start) + else: + fabtools.service.reload('apache2') + + +@do_verbose +def apache_restart(): + """ Starts/Restarts apache """ + + if not apache_started(): + fabric.api.execute(apache_start) + else: + fabtools.service.restart('apache2') + + +@do_verbose +def apache_started(): + """ Returns true/false depending on apache service is started """ + + return fabtools.service.is_running('apache2') + + +@do_verbose +def apache_confirm_start(): + """ Confirms launch of apache """ + + current_role = pydiploy.prepare._get_current_role() + if fabric.contrib.console.confirm( + "\napache on %s (role %s) seems not to be started ! \ + \n\nDo you want to try to start it?" % + (fabric.colors.red(env.host), + fabric.colors.red(current_role)), default=False): + fabtools.service.start('apache2') + + +@do_verbose +def web_configuration(): + """ Setups webserver's configuration """ + + fabric.api.execute(up_site_config) + fabric.api.execute(down_site_config) + fabric.api.execute(apache_disable_site, '000-default') + + +@do_verbose +def up_site_config(): + """ Uploads site config for apache """ + + apache_root = '/etc/apache2' + apache_available = os.path.join(apache_root, 'sites-available') + apache_enabled = os.path.join(apache_root, 'sites-enabled') + app_conf = os.path.join(apache_available, '%s.conf' % env.server_name) + + fabtools.files.upload_template('apache_host.conf.tpl', + app_conf, + context=env, + template_dir=os.path.join( + env.lib_path, 'templates'), + use_jinja=True, + use_sudo=True, + user='root', + chown=True, + mode='644') + + fabric.api.execute(apache_enable_site, env.server_name) + + +@do_verbose +def down_site_config(): + """ Uploads site_down config for apache """ + + apache_root = '/etc/apache2' + apache_available = os.path.join(apache_root, 'sites-available') + app_conf = os.path.join(apache_available, '%s_down.conf' % env.server_name) + + fabtools.files.upload_template('apache_host_down.conf.tpl', + app_conf, + context=env, + template_dir=os.path.join( + env.lib_path, 'templates'), + use_jinja=True, + use_sudo=True, + user='root', + chown=True, + mode='644') + + # fabric.api.execute(upload_maintenance_page) + + +@do_verbose +def set_website_up(): + """ Sets website up """ + + # apache_root = '/etc/apache2' + # apache_available = os.path.join(apache_root, 'sites-available') + # apache_enabled = os.path.join(apache_root, 'sites-enabled') + # app_conf = os.path.join(apache_available, '%s.conf' % env.server_name) + + # if not fabtools.files.is_file(app_conf): + # fabric.api.execute(up_site_config) + + # if fabtools.files.is_link('%s/%s_down.conf' % (apache_enabled, + # env.server_name)): + # with fabric.api.cd(apache_enabled): + # fabric.api.sudo('rm -f %s_down.conf' % env.server_name) + + # if not fabtools.files.is_link('%s/%s.conf' % (apache_enabled, + # env.server_name)): + # with fabric.api.cd(apache_enabled): + # fabric.api.sudo('ln -s %s .' % app_conf) + + fabric.api.execute(apache_disable_site, '%s_down' % env.server_name) + fabric.api.execute(apache_enable_site, '%s' % env.server_name) + + +@do_verbose +def set_website_down(): + """ Sets website down """ + + # apache_root = '/etc/apache2' + # apache_available = os.path.join(apache_root, 'sites-available') + # apache_enabled = os.path.join(apache_root, 'sites-enabled') + # app_down_conf = os.path.join( + # apache_available, '%s_down.conf' % env.server_name) + + # if not fabtools.files.is_file(app_down_conf): + # fabric.api.execute(down_site_config) + + # if fabtools.files.is_link('%s/%s.conf' % (apache_enabled, + # env.server_name)): + # with fabric.api.cd(apache_enabled): + # fabric.api.sudo('rm -f %s.conf' % env.server_name) + + # if not fabtools.files.is_link('%s/%s_down.conf' % (apache_enabled, + # env.server_name)): + # with fabric.api.cd(apache_enabled): + # fabric.api.sudo('ln -s %s .' % app_down_conf) + + apache_disable_site('%s' % env.server_name) + apache_enable_site('%s_down' % env.server_name) + # fabric.api.execute(apache_reload) + + +@do_verbose +def upload_maintenance_page(): + """ Uploads and forges maintenance.html according to template """ + + maintenance_file = os.path.join(env.remote_static_root, + env.application_name, + 'maintenance.html') + vars_required = ['maintenance_text', 'maintenance_title'] + + for v in vars_required: + if v in env: + env[v] = env[v].decode('utf-8') + + fabtools.files.upload_template('maintenance.html.tpl', + maintenance_file, + context=env, + template_dir=os.path.join( + env.lib_path, 'templates'), + use_jinja=True, + use_sudo=True, + user='root', + chown=True, + mode='644') + + +@do_verbose +def enable_apache_modules(): + """ Enables one or many apache modules needed """ + + if env.has_key('extra_apache_enabled_modules'): + for apache_module in env.extra_apache_modules: + fabtools.require.apache.module_enabled(apache_module) + + +@do_verbose +def disable_apache_modules(): + """ Enables one or many apache modules needed """ + + if env.has_key('extra_apache_disabled_modules'): + for apache_module in env.extra_apache_modules: + fabtools.require.apache.module_disabled(apache_module) + + +@do_verbose +def apache_enable_site(site=None, restart=False): + """ Enables a site (eg : a2enssite site) """ + + if site: + fabtools.require.apache.site_enabled( + site) if restart else fabtools.require.apache.site_enabled(site) + + +@do_verbose +def apache_disable_site(site=None, restart=False): + """ Disables a site (eg : a2dissite site) """ + + if site: + fabtools.require.apache.site_disabled(site) diff --git a/pydiploy/templates/apache_host.conf.tpl b/pydiploy/templates/apache_host.conf.tpl new file mode 100644 index 0000000..e6cad41 --- /dev/null +++ b/pydiploy/templates/apache_host.conf.tpl @@ -0,0 +1,18 @@ +{% if apache_custom_vhost %} +{{ apache_custom_vhost }} +{% else %} + + ServerName {{ server_name }} + + DocumentRoot {{ remote_home }}/{{ server_name }}/current + + + Options Indexes FollowSymLinks MultiViews + + AllowOverride All + + Order allow,deny + allow from all + + +{% endif %} \ No newline at end of file diff --git a/pydiploy/templates/apache_host_down.conf.tpl b/pydiploy/templates/apache_host_down.conf.tpl new file mode 100644 index 0000000..048a70a --- /dev/null +++ b/pydiploy/templates/apache_host_down.conf.tpl @@ -0,0 +1,14 @@ + + ServerName {{ server_name }} + + DocumentRoot {{ remote_home }}/{{ server_name }}/current + + + Options Indexes FollowSymLinks MultiViews + + AllowOverride All + + Order allow,deny + allow from all + + diff --git a/tools/pydiployfabfile.sublime-snippet b/tools/pydiployfabfile.sublime-snippet index ac93dea..4d8c72e 100644 --- a/tools/pydiployfabfile.sublime-snippet +++ b/tools/pydiployfabfile.sublime-snippet @@ -75,12 +75,17 @@ env.keep_releases = 2 # number of old releases to keep before cleaning # env.circus_package_name = 'https://github.com/githubaccount/circus/archive/master.zip' # change the package to use to install circus # env.no_circus_web = True # true if you don't want circus-web +# env.circus_backend = 'gevent' # name of circus backend to use + +# env.chaussette_backend = 'waitress' # name of chaussette backend to use. You need to add this backend in the app requirement file. # env.nginx_location_extra_directives = ['proxy_read_timeout 120'] # add directive(s) to nginx config file in location part # env.nginx_start_confirmation = True # if True when nginx is not started needs confirmation to start it. # env.socket_host='localhost' # use it in env method to force a socket host +# env.run_tests_command = 'tox' + # fill and uncomment not to pass parameters in term (eg: fab tag:master test --set default_db_host='localhost',default_db_name='my_app_db' ) # env.default_db_host = 'localhost' # env.default_db_name = 'myapp_db' @@ -101,6 +106,7 @@ def test(): env.short_server_name = 'myapp-dev' env.static_folder = '/site_media/' env.server_ip = '192.168.1.3' + #env.port = '1664' env.no_shared_sessions = False env.server_ssl_on = False env.goal = 'test' @@ -125,6 +131,7 @@ def prod(): env.short_server_name = 'myapp' env.static_folder = '/site_media/' env.server_ip = '' + #env.port = '' env.no_shared_sessions = False env.server_ssl_on = True env.path_to_cert = '/etc/ssl/certs/myapp.net.pem' @@ -196,7 +203,7 @@ def deploy(): @task def deploy_backend(update_pkg=False): """Deploy code on server""" - execute(pydiploy_deploy_backend) + execute(pydiploy_deploy_backend, update_pkg) @roles('lb') @@ -291,4 +298,4 @@ def custom_manage_cmd(cmd): Pydiploy fabfile 1.1.4.0 pydiployfab source.python - + \ No newline at end of file From ebdfcc23a11009210be9cd58df3502bfe4f8adb9 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Mon, 8 Oct 2018 14:17:40 +0200 Subject: [PATCH 02/44] Monkey patch for vistualenv creation when using sudo --- pydiploy/require/python/virtualenv.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/pydiploy/require/python/virtualenv.py b/pydiploy/require/python/virtualenv.py index 6432713..ed730b7 100644 --- a/pydiploy/require/python/virtualenv.py +++ b/pydiploy/require/python/virtualenv.py @@ -1,12 +1,19 @@ # -*- coding: utf-8 -*- """ This module is used for virtualenv relatives """ - +from contextlib import contextmanager import fabtools from fabric.api import env from pydiploy.decorators import do_verbose +@contextmanager +def shell(new_shell): + old_shell, env.shell = env.shell, new_shell + yield + env.shell = old_shell + + @do_verbose def virtualenv(clear=False): """ Creates virtualenv """ @@ -15,7 +22,8 @@ def virtualenv(clear=False): group=env.remote_group, use_sudo=True) python_bin = '/usr/bin/python%s' % env.remote_python_version - fabtools.require.python.virtualenv(env.remote_virtualenv_dir, - user=env.remote_owner, - clear=clear, use_sudo=True, - venv_python=python_bin) + with shell("HOME=~%s %s" % (env.remote_owner, env.shell)): + fabtools.require.python.virtualenv(env.remote_virtualenv_dir, + user=env.remote_owner, + clear=clear, use_sudo=True, + venv_python=python_bin) From f162ec4a3f3541b3536f2d0afb3e432d01e87086 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Tue, 24 May 2016 17:03:26 +0200 Subject: [PATCH 03/44] change version number in sb snippets --- tools/pydiployfabfilebottle.sublime-snippet | 4 ++-- tools/pydiployfabfilesimple.sublime-snippet | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/pydiployfabfilebottle.sublime-snippet b/tools/pydiployfabfilebottle.sublime-snippet index 8792c7b..f969ce0 100644 --- a/tools/pydiployfabfilebottle.sublime-snippet +++ b/tools/pydiployfabfilebottle.sublime-snippet @@ -292,7 +292,7 @@ def set_up(): execute(pydiploy_set_up) ]]> -Pydiploy fabfile bottle 1.1.6.0 +Pydiploy fabfile bottle 1.1.7.0 pydiployfabbottle source.python - \ No newline at end of file + diff --git a/tools/pydiployfabfilesimple.sublime-snippet b/tools/pydiployfabfilesimple.sublime-snippet index e99659b..8f4037a 100644 --- a/tools/pydiployfabfilesimple.sublime-snippet +++ b/tools/pydiployfabfilesimple.sublime-snippet @@ -161,7 +161,7 @@ def post_install_backend(): execute(pydiploy_postinstall_backend) ]]> -Pydiploy fabfile simple 1.1.6.0 +Pydiploy fabfile simple 1.1.7.0 pydiployfabsimple source.python - \ No newline at end of file + From d79bde766d915f2470a84a876aeaff14add73048 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Wed, 25 May 2016 09:56:30 +0200 Subject: [PATCH 04/44] Add autoupgrade of pip in venv --- pydiploy/require/python/utils.py | 14 +++++++++++--- tests/test_require_python.py | 3 ++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pydiploy/require/python/utils.py b/pydiploy/require/python/utils.py index a5d0b90..b652dcf 100644 --- a/pydiploy/require/python/utils.py +++ b/pydiploy/require/python/utils.py @@ -30,8 +30,8 @@ def application_dependencies(upgrade_pkg, staging=True): requirements_file = os.path.join('requirements', '%s.txt' % env.goal) if staging else 'requirements.txt' # ugly fix for error when pip install fail and error raises while /home/user/.pip not writable - pip_log = "%s/pip_error.log " % env.remote_home - pip_cmd = 'pip --no-cache-dir --log-file %s' % pip_log + pip_log = '%s/pip_error.log' % env.remote_home + pip_cmd = 'pip --log-file %s' % pip_log if 'oracle_client_version' in env: oracle_dir = 'instantclient_%s' % '_'.join( env.oracle_client_version.split('.')[:2]) @@ -41,11 +41,19 @@ def application_dependencies(upgrade_pkg, staging=True): env.remote_home, oracle_root_path) pip_cmd = 'ORACLE_HOME=%s pip' % oracle_full_path + # upgrade pip to latest version + fabtools.require.python.install('pip', + upgrade=True, + use_sudo=True, + user=env.remote_owner, + pip_cmd=pip_cmd, + quiet=True) + fabtools.python.install_requirements(requirements_file, use_sudo=True, user=env.remote_owner, upgrade=upgrade_pkg, - pip_cmd=pip_cmd, + pip_cmd='%s --no-cache-dir' % pip_cmd, quiet=True) fabric.api.sudo( diff --git a/tests/test_require_python.py b/tests/test_require_python.py index 6fbfed8..f9adfae 100644 --- a/tests/test_require_python.py +++ b/tests/test_require_python.py @@ -42,9 +42,10 @@ def test_python_pkg(self, python_install, deb_packages): @patch('fabtools.python.virtualenv', return_value=Mock()) @patch('fabric.api.cd', return_value=Mock()) + @patch('fabtools.require.python.install', return_value=Mock()) @patch('fabric.api.sudo', return_value=Mock()) @patch('fabtools.python.install_requirements', return_value=Mock()) - def test_application_dependencies(self, install_requirements, api_sudo, api_cd, python_virtualenv): + def test_application_dependencies(self, install_requirements, api_sudo, python_install, api_cd, python_virtualenv): python_virtualenv.return_value.__exit__ = Mock() python_virtualenv.return_value.__enter__ = Mock() From 421c2d391d5b566a368ddb66a49e8dccee406679 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Wed, 25 May 2016 09:59:09 +0200 Subject: [PATCH 05/44] Release 1.1.7.1 --- pydiploy/version.py | 2 +- tools/pydiployfabfile.sublime-snippet | 2 +- tools/pydiployfabfilebottle.sublime-snippet | 2 +- tools/pydiployfabfilesimple.sublime-snippet | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pydiploy/version.py b/pydiploy/version.py index a85bfe9..33f8c25 100644 --- a/pydiploy/version.py +++ b/pydiploy/version.py @@ -1,3 +1,3 @@ # pydiploy version -__version__ = '1.1.7.0' +__version__ = '1.1.7.1' __version_info__ = tuple([ int(num) for num in __version__.split('.')]) diff --git a/tools/pydiployfabfile.sublime-snippet b/tools/pydiployfabfile.sublime-snippet index 99b1a3e..26cbdf5 100644 --- a/tools/pydiployfabfile.sublime-snippet +++ b/tools/pydiployfabfile.sublime-snippet @@ -301,7 +301,7 @@ def custom_manage_cmd(cmd): execute(pydiploy_custom_command,cmd) ]]> -Pydiploy fabfile 1.1.7.0 +Pydiploy fabfile 1.1.7.1 pydiployfab source.python diff --git a/tools/pydiployfabfilebottle.sublime-snippet b/tools/pydiployfabfilebottle.sublime-snippet index f969ce0..cd634fc 100644 --- a/tools/pydiployfabfilebottle.sublime-snippet +++ b/tools/pydiployfabfilebottle.sublime-snippet @@ -292,7 +292,7 @@ def set_up(): execute(pydiploy_set_up) ]]> -Pydiploy fabfile bottle 1.1.7.0 +Pydiploy fabfile bottle 1.1.7.1 pydiployfabbottle source.python diff --git a/tools/pydiployfabfilesimple.sublime-snippet b/tools/pydiployfabfilesimple.sublime-snippet index 8f4037a..40d9d5a 100644 --- a/tools/pydiployfabfilesimple.sublime-snippet +++ b/tools/pydiployfabfilesimple.sublime-snippet @@ -161,7 +161,7 @@ def post_install_backend(): execute(pydiploy_postinstall_backend) ]]> -Pydiploy fabfile simple 1.1.7.0 +Pydiploy fabfile simple 1.1.7.1 pydiployfabsimple source.python From d8912120cf1d6776b04bf1dd6eddf6af59c2858c Mon Sep 17 00:00:00 2001 From: Morgan Date: Mon, 5 Sep 2016 17:12:33 +0200 Subject: [PATCH 06/44] systemd conf and service for circus, patch symbolic link for localtime --- pydiploy/require/circus.py | 64 +++++++++++++++++++++------ pydiploy/require/system.py | 5 ++- pydiploy/templates/circus.service.tpl | 13 ++++++ 3 files changed, 67 insertions(+), 15 deletions(-) create mode 100644 pydiploy/templates/circus.service.tpl diff --git a/pydiploy/require/circus.py b/pydiploy/require/circus.py index bf9fcb1..501221f 100644 --- a/pydiploy/require/circus.py +++ b/pydiploy/require/circus.py @@ -17,7 +17,7 @@ import fabric import fabtools -from fabric.api import env +from fabric.api import env, warn_only from pydiploy.decorators import do_verbose @@ -95,26 +95,62 @@ def upstart(): """ # TODO: implement as systemd service !!! - # init files to declare circus as an upstart daemon - fabtools.files.upload_template('upstart.conf.tpl', - '/etc/init/circus.conf', - context=env, - template_dir=os.path.join( - env.lib_path, 'templates'), - use_sudo=True, - user='root', - chown=True, - mode='644', - use_jinja=True) + + # Systemd + if is_systemd(): + # init files to declare circus as a systemd daemon + fabtools.files.upload_template('circus.service.tpl', + '/etc/systemd/system/circus.service', + context=env, + template_dir=os.path.join( + env.lib_path, 'templates'), + use_sudo=True, + user='root', + chown=True, + mode='644', + use_jinja=True) + fabric.api.sudo('systemctl daemon-reload') + # Upstart + else: + + # init files to declare circus as an upstart daemon + fabtools.files.upload_template('upstart.conf.tpl', + '/etc/init/circus.conf', + context=env, + template_dir=os.path.join( + env.lib_path, 'templates'), + use_sudo=True, + user='root', + chown=True, + mode='644', + use_jinja=True) @do_verbose def app_reload(): """ Starts/restarts app using circus """ - if not 'running' in fabric.api.sudo('status circus'): - fabric.api.sudo('start circus') + # Systemd + if is_systemd(): + start_cmd = 'systemctl start circus.service' + status_cmd = 'systemctl is-active circus.service' + with warn_only(): + running = 'inactive' not in fabric.api.sudo(status_cmd) + # Upstart + else: + start_cmd = 'start circus' + status_cmd = 'status circus' + running = 'running' in fabric.api.sudo(status_cmd) + + if not running: + fabric.api.sudo(start_cmd) else: with fabric.api.settings(sudo_user=env.remote_owner): fabric.api.sudo('circusctl reloadconfig') fabric.api.sudo('circusctl restart %s' % env.application_name) + + +@do_verbose +def is_systemd(): + """ return True if systemd is used """ + return fabtools.files.is_dir("/run/systemd/system") diff --git a/pydiploy/require/system.py b/pydiploy/require/system.py index 614a00d..b36ba35 100644 --- a/pydiploy/require/system.py +++ b/pydiploy/require/system.py @@ -73,7 +73,10 @@ def set_timezone(): print("Cannot deploy to non-debian/ubuntu host") return - return fabric.api.sudo("cp -f /usr/share/zoneinfo/%s /etc/localtime" % env.timezone) + if fabtools.files.is_link("/etc/localtime"): + return fabric.api.sudo("ln -sf /usr/share/zoneinfo/%s /etc/localtime" % env.timezone) + else: + return fabric.api.sudo("cp -f /usr/share/zoneinfo/%s /etc/localtime" % env.timezone) @do_verbose diff --git a/pydiploy/templates/circus.service.tpl b/pydiploy/templates/circus.service.tpl new file mode 100644 index 0000000..41b1f14 --- /dev/null +++ b/pydiploy/templates/circus.service.tpl @@ -0,0 +1,13 @@ +[Unit] +Description=Circus process manager +After=syslog.target network.target nss-lookup.target + +[Service] +Type=simple +ExecReload=/usr/local/bin/circusctl reload +ExecStart=/usr/local/bin/circusd {{ remote_home }}/.circus.ini +Restart=always +RestartSec=5 + +[Install] +WantedBy=default.target From 8947861b5bca104866d3b405e128ee32d8cd4333 Mon Sep 17 00:00:00 2001 From: Morgan Date: Tue, 6 Sep 2016 10:32:54 +0200 Subject: [PATCH 07/44] correct unit tests --- pydiploy/require/circus.py | 4 ---- tests/test_require_circus.py | 6 ++++-- tests/test_require_system.py | 3 ++- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pydiploy/require/circus.py b/pydiploy/require/circus.py index 501221f..3d4fb9b 100644 --- a/pydiploy/require/circus.py +++ b/pydiploy/require/circus.py @@ -93,9 +93,6 @@ def upstart(): """ Sets script to start circus at boot using templates in templates dir """ - - # TODO: implement as systemd service !!! - # Systemd if is_systemd(): # init files to declare circus as a systemd daemon @@ -112,7 +109,6 @@ def upstart(): fabric.api.sudo('systemctl daemon-reload') # Upstart else: - # init files to declare circus as an upstart daemon fabtools.files.upload_template('upstart.conf.tpl', '/etc/init/circus.conf', diff --git a/tests/test_require_circus.py b/tests/test_require_circus.py index b82fe78..2fd3601 100644 --- a/tests/test_require_circus.py +++ b/tests/test_require_circus.py @@ -113,8 +113,9 @@ def test_app_circus_conf(self, upload_template): self.assertTrue( str(upload_template.call_args).find("user='remote_owner'") > 0) + @patch('fabtools.files.is_dir', return_value=False) @patch('fabtools.files.upload_template', return_value=Mock()) - def test_upstart(self, upload_template): + def test_upstart(self, upload_template, is_systemd): upstart() self.assertTrue(upload_template.called) @@ -126,9 +127,10 @@ def test_upstart(self, upload_template): .find("template_dir='lib_path/templates'") > 0) self.assertTrue(str(upload_template.call_args).find("user='root'") > 0) + @patch('fabtools.files.is_dir', return_value=False) @patch('fabric.api.settings', return_value=Mock()) @patch('fabric.api.sudo', return_value='running') - def test_app_reload(self, api_sudo, api_settings): + def test_app_reload(self, api_sudo, api_settings, is_systemd): api_settings.return_value.__exit__ = Mock() api_settings.return_value.__enter__ = Mock() diff --git a/tests/test_require_system.py b/tests/test_require_system.py index ed08b58..cf19d11 100644 --- a/tests/test_require_system.py +++ b/tests/test_require_system.py @@ -78,9 +78,10 @@ def test_set_locale(self, api_sudo, api_run): self.assertEqual(api_sudo.call_args_list, [ call('locale-gen FR_fr'), call('/usr/sbin/update-locale LANG=FR_fr')]) + @patch('fabtools.files.is_link', return_value=False) @patch('fabtools.system.distrib_id', return_value='Notsupported') @patch('fabric.api.sudo', return_value=Mock()) - def test_set_timezone(self, api_sudo, distrib_id): + def test_set_timezone(self, api_sudo, distrib_id, is_link): set_timezone() # test error self.assertTrue(distrib_id.called) From 37eb90588178d4a78793098ab649e95860f87895 Mon Sep 17 00:00:00 2001 From: Morgan Date: Tue, 6 Sep 2016 10:51:43 +0200 Subject: [PATCH 08/44] systemd and nginx --- pydiploy/require/circus.py | 7 +------ pydiploy/require/nginx.py | 23 ++++++++++++++++++----- pydiploy/require/system.py | 6 ++++++ tests/test_require_nginx.py | 13 ++++++++----- 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/pydiploy/require/circus.py b/pydiploy/require/circus.py index 3d4fb9b..3520c40 100644 --- a/pydiploy/require/circus.py +++ b/pydiploy/require/circus.py @@ -19,6 +19,7 @@ import fabtools from fabric.api import env, warn_only from pydiploy.decorators import do_verbose +from .system import is_systemd @do_verbose @@ -144,9 +145,3 @@ def app_reload(): with fabric.api.settings(sudo_user=env.remote_owner): fabric.api.sudo('circusctl reloadconfig') fabric.api.sudo('circusctl restart %s' % env.application_name) - - -@do_verbose -def is_systemd(): - """ return True if systemd is used """ - return fabtools.files.is_dir("/run/systemd/system") diff --git a/pydiploy/require/nginx.py b/pydiploy/require/nginx.py index 1ab5886..fb1c2c7 100644 --- a/pydiploy/require/nginx.py +++ b/pydiploy/require/nginx.py @@ -9,6 +9,7 @@ import pydiploy from fabric.api import env from pydiploy.decorators import do_verbose +from .system import is_systemd @do_verbose @@ -33,7 +34,10 @@ def nginx_start(): if not nginx_started() and ('nginx_force_start' not in env or not env.nginx_force_start): fabric.api.puts("Nginx is not started") else: - fabtools.service.start('nginx') + if is_systemd(): + fabtools.systemd.start('nginx') + else: + fabtools.service.start('nginx') @@ -47,7 +51,10 @@ def nginx_reload(): else: fabric.api.puts("Nginx is not started") else: - fabtools.service.reload('nginx') + if is_systemd(): + fabtools.systemd.reload('nginx') + else: + fabtools.service.reload('nginx') @do_verbose @@ -60,14 +67,20 @@ def nginx_restart(): else: fabric.api.puts("Nginx is not started") else: - fabtools.service.restart('nginx') + if is_systemd(): + fabtools.systemd.restart('nginx') + else: + fabtools.service.restart('nginx') @do_verbose def nginx_started(): """ Returns true/false depending on nginx service is started """ - - return fabtools.service.is_running('nginx') + if is_systemd(): + # return fabtools.systemd.is_running('nginx') + return 'inactive' not in fabric.api.sudo('systemctl is-active nginx.service') + else: + return fabtools.service.is_running('nginx') @do_verbose diff --git a/pydiploy/require/system.py b/pydiploy/require/system.py index b36ba35..386c880 100644 --- a/pydiploy/require/system.py +++ b/pydiploy/require/system.py @@ -139,3 +139,9 @@ def install_extra_ppa(extra_ppa): """ for ppa in extra_ppa: fabtools.require.deb.ppa(ppa) + + +@do_verbose +def is_systemd(): + """ return True if systemd is used """ + return fabtools.files.is_dir("/run/systemd/system") diff --git a/tests/test_require_nginx.py b/tests/test_require_nginx.py index 5aa9478..1729bd9 100644 --- a/tests/test_require_nginx.py +++ b/tests/test_require_nginx.py @@ -46,9 +46,10 @@ def test_nginx_pkg(self, deb_packages): self.assertTrue(deb_packages.called) self.assertEqual(deb_packages.call_args, call(['nginx'], update=False)) + @patch('fabtools.files.is_dir', return_value=False) @patch('fabtools.service.is_running', return_value=False) @patch('fabtools.service.start', return_value=Mock()) - def test_nginx_start(self, service_start, is_running): + def test_nginx_start(self, service_start, is_running, is_systemd): # is_running False + Force start to false env.nginx_force_start = False nginx_start() @@ -77,10 +78,11 @@ def test_nginx_start(self, service_start, is_running): nginx_start() self.assertTrue(service_start.called) + @patch('fabtools.files.is_dir', return_value=False) @patch('fabtools.service.is_running', return_value=True) @patch('fabtools.service.start', return_value=Mock()) @patch('fabtools.service.reload', return_value=Mock()) - def test_nginx_reload(self, reload, start, is_running): + def test_nginx_reload(self, reload, start, is_running, is_systemd): # Nginx run nginx_reload() self.assertTrue(reload.called) @@ -105,11 +107,11 @@ def test_nginx_reload(self, reload, start, is_running): self.assertEqual(start.call_args, call('nginx')) - + @patch('fabtools.files.is_dir', return_value=False) @patch('fabtools.service.is_running', return_value=True) @patch('fabtools.service.start', return_value=Mock()) @patch('fabtools.service.restart', return_value=Mock()) - def test_nginx_restart(self, restart, start, is_running): + def test_nginx_restart(self, restart, start, is_running, is_systemd): # Nginx run nginx_restart() self.assertTrue(is_running.called) @@ -132,8 +134,9 @@ def test_nginx_restart(self, restart, start, is_running): self.assertTrue(start.called) self.assertEqual(start.call_args, call('nginx')) + @patch('fabtools.files.is_dir', return_value=False) @patch('fabtools.service.is_running', return_value=True) - def test_nginx_started(self, is_running): + def test_nginx_started(self, is_running, is_systemd): res = nginx_started() self.assertTrue(is_running.called) self.assertEqual(res, True) From 201220a72b8ea33a7c6a92d41efceebbc74f7d79 Mon Sep 17 00:00:00 2001 From: Morgan Date: Tue, 6 Sep 2016 14:33:47 +0200 Subject: [PATCH 09/44] add unit test for systemd circus+chaussette and timezone link --- tests/test_require_circus.py | 46 +++++++++++++++++ tests/test_require_nginx.py | 97 ++++++++++++++++++++++++++++++++++++ tests/test_require_system.py | 18 +++++++ 3 files changed, 161 insertions(+) diff --git a/tests/test_require_circus.py b/tests/test_require_circus.py index 2fd3601..74241da 100644 --- a/tests/test_require_circus.py +++ b/tests/test_require_circus.py @@ -127,6 +127,22 @@ def test_upstart(self, upload_template, is_systemd): .find("template_dir='lib_path/templates'") > 0) self.assertTrue(str(upload_template.call_args).find("user='root'") > 0) + @patch('fabric.api.sudo', return_value=Mock()) + @patch('fabtools.files.is_dir', return_value=True) + @patch('fabtools.files.upload_template', return_value=Mock()) + def test_upstart_systemd(self, upload_template, is_systemd, mock_reload_daemon): + upstart() + + self.assertTrue(upload_template.called) + self.assertTrue( + str(upload_template.call_args).find("'circus.service.tpl'") > 0) + self.assertTrue( + str(upload_template.call_args).find("'/etc/systemd/system/circus.service'") > 0) + self.assertTrue(str(upload_template.call_args) + .find("template_dir='lib_path/templates'") > 0) + self.assertTrue(str(upload_template.call_args).find("user='root'") > 0) + self.assertTrue(mock_reload_daemon.called) + @patch('fabtools.files.is_dir', return_value=False) @patch('fabric.api.settings', return_value=Mock()) @patch('fabric.api.sudo', return_value='running') @@ -156,3 +172,33 @@ def test_app_reload(self, api_sudo, api_settings, is_systemd): self.assertTrue(api_sudo.called) self.assertEqual(api_sudo.call_args, call('start circus')) self.assertFalse(api_settings.called) + + @patch('fabtools.files.is_dir', return_value=True) + @patch('fabric.api.settings', return_value=Mock()) + @patch('fabric.api.sudo', return_value='active') + def test_app_reload_systemd(self, api_sudo, api_settings, is_systemd): + + api_settings.return_value.__exit__ = Mock() + api_settings.return_value.__enter__ = Mock() + + # test if running + app_reload() + + self.assertTrue(api_sudo.called) + self.assertEqual(api_sudo.call_args_list, [call('systemctl is-active circus.service'), call( + 'circusctl reloadconfig'), call('circusctl restart application_name')]) + + self.assertTrue(api_settings.called) + self.assertEqual( + api_settings.call_args, call(sudo_user='remote_owner')) + + # test if not running + api_sudo.return_value = 'inactive' + api_sudo.called = False + api_settings.called = False + + app_reload() + + self.assertTrue(api_sudo.called) + self.assertEqual(api_sudo.call_args, call('systemctl start circus.service')) + self.assertFalse(api_settings.called) diff --git a/tests/test_require_nginx.py b/tests/test_require_nginx.py index 1729bd9..542d952 100644 --- a/tests/test_require_nginx.py +++ b/tests/test_require_nginx.py @@ -78,6 +78,38 @@ def test_nginx_start(self, service_start, is_running, is_systemd): nginx_start() self.assertTrue(service_start.called) + @patch('fabtools.files.is_dir', return_value=True) + @patch('fabric.api.sudo', return_value="inactive") + @patch('fabtools.systemd.start', return_value=Mock()) + def test_nginx_start_systemd(self, service_start, is_active, is_systemd): + # is_running False + Force start to false + env.nginx_force_start = False + nginx_start() + self.assertFalse(service_start.called) + # is_running False + No force start option + del env['nginx_force_start'] + nginx_start() + self.assertFalse(service_start.called) + # is_running False + Force start to True + env.nginx_force_start = True + nginx_start() + self.assertTrue(service_start.called) + + is_active.return_value = "active" + + # is_running True + Force start to false + env.nginx_force_start = False + nginx_start() + self.assertTrue(service_start.called) + # is_running True + No force start option + del env['nginx_force_start'] + nginx_start() + self.assertTrue(service_start.called) + # is_running True + Force start to True + env.nginx_force_start = True + nginx_start() + self.assertTrue(service_start.called) + @patch('fabtools.files.is_dir', return_value=False) @patch('fabtools.service.is_running', return_value=True) @patch('fabtools.service.start', return_value=Mock()) @@ -106,6 +138,33 @@ def test_nginx_reload(self, reload, start, is_running, is_systemd): self.assertTrue(start.called) self.assertEqual(start.call_args, call('nginx')) + @patch('fabtools.files.is_dir', return_value=True) + @patch('fabric.api.sudo', return_value="active") + @patch('fabtools.systemd.start', return_value=Mock()) + @patch('fabtools.systemd.reload', return_value=Mock()) + def test_nginx_reload_systemd(self, reload, start, is_active, is_systemd): + # Nginx run + nginx_reload() + self.assertTrue(reload.called) + self.assertEqual(reload.call_args, call('nginx')) + self.assertTrue(is_active.called) + self.assertFalse(start.called) + # Nginx stopped + is_active.return_value = "inactive" + reload.called = False + is_active.called = False + start.called = False + nginx_reload() + self.assertTrue(is_active.called) + self.assertFalse(reload.called) + self.assertFalse(start.called) + # Force reload + env.nginx_force_start = True + nginx_reload() + self.assertTrue(is_active.called) + self.assertFalse(reload.called) + self.assertTrue(start.called) + self.assertEqual(start.call_args, call('nginx')) @patch('fabtools.files.is_dir', return_value=False) @patch('fabtools.service.is_running', return_value=True) @@ -134,6 +193,33 @@ def test_nginx_restart(self, restart, start, is_running, is_systemd): self.assertTrue(start.called) self.assertEqual(start.call_args, call('nginx')) + @patch('fabtools.files.is_dir', return_value=True) + @patch('fabric.api.sudo', return_value="active") + @patch('fabtools.systemd.start', return_value=Mock()) + @patch('fabtools.systemd.restart', return_value=Mock()) + def test_nginx_restart_systemd(self, restart, start, is_active, is_systemd): + # Nginx run + nginx_restart() + self.assertTrue(is_active.called) + self.assertFalse(start.called) + self.assertTrue(restart.called) + # Nginx stopped + is_active.return_value = "inactive" + restart.called = False + is_active.called = False + start.called = False + nginx_restart() + self.assertFalse(restart.called) + self.assertTrue(is_active.called) + self.assertFalse(start.called) + # Force reload + env.nginx_force_start = True + nginx_restart() + self.assertFalse(restart.called) + self.assertTrue(is_active.called) + self.assertTrue(start.called) + self.assertEqual(start.call_args, call('nginx')) + @patch('fabtools.files.is_dir', return_value=False) @patch('fabtools.service.is_running', return_value=True) def test_nginx_started(self, is_running, is_systemd): @@ -145,6 +231,17 @@ def test_nginx_started(self, is_running, is_systemd): self.assertTrue(is_running.called) self.assertEqual(res, False) + @patch('fabtools.files.is_dir', return_value=True) + @patch('fabric.api.sudo', return_value="active") + def test_nginx_started_systemd(self, is_active, is_systemd): + res = nginx_started() + self.assertTrue(is_active.called) + self.assertEqual(res, True) + is_active.return_value = "inactive" + res = nginx_started() + self.assertTrue(is_active.called) + self.assertEqual(res, False) + @patch('fabric.contrib.project.rsync_project', return_value=Mock()) def test_web_static_files(self, rsync_project): web_static_files() diff --git a/tests/test_require_system.py b/tests/test_require_system.py index cf19d11..d89d538 100644 --- a/tests/test_require_system.py +++ b/tests/test_require_system.py @@ -96,6 +96,24 @@ def test_set_timezone(self, api_sudo, distrib_id, is_link): self.assertEqual(api_sudo.call_args, call( 'cp -f /usr/share/zoneinfo/mytimezone /etc/localtime')) + @patch('fabtools.files.is_link', return_value=True) + @patch('fabtools.system.distrib_id', return_value='Notsupported') + @patch('fabric.api.sudo', return_value=Mock()) + def test_set_timezone_link(self, api_sudo, distrib_id, is_link): + set_timezone() + # test error + self.assertTrue(distrib_id.called) + self.assertFalse(api_sudo.called) + + # test if it works + distrib_id.return_value = 'Ubuntu' + set_timezone() + self.assertTrue(distrib_id.called) + self.assertTrue(api_sudo.called) + + self.assertEqual(api_sudo.call_args, call( + 'ln -sf /usr/share/zoneinfo/mytimezone /etc/localtime')) + @patch('fabric.api.sudo', return_value=Mock()) def test_permissions(self, api_sudo): permissions() From 2a60f8f4c3b65e7741f40f5bc6f299d2504337d4 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Tue, 6 Sep 2016 16:44:01 +0200 Subject: [PATCH 10/44] Add way to set debian source on remote server --- README.rst | 4 ++++ pydiploy/bottle.py | 4 ++++ pydiploy/django.py | 3 +++ pydiploy/params.py | 3 +++ pydiploy/require/system.py | 16 +++++++++++++--- pydiploy/simple.py | 4 ++++ tests/test_bottle.py | 7 +++++++ tests/test_django.py | 13 ++++++++----- tests/test_releases_manager.py | 1 + tests/test_require_system.py | 16 ++++++++++++---- tests/test_simple.py | 7 +++++++ tools/pydiployfabfile.sublime-snippet | 1 + tools/pydiployfabfilebottle.sublime-snippet | 1 + tools/pydiployfabfilesimple.sublime-snippet | 1 + 14 files changed, 69 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 96fd9d8..2eee4de 100644 --- a/README.rst +++ b/README.rst @@ -227,6 +227,10 @@ Optional parameters extra_ppa_to_install = ['ppa:vincent-c/ponysay'] + - extra_source_to_install adds extra(s) debian sourc(e)s when setting server : :: + + extra_source_to_install = [['mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'], ['deb-src', 'http://site.example.com/debian', 'distribution', 'component1', 'component2', 'component3']] + - extra_pkg_to_install adds extra(s) package(s) when setting server : :: env.extra_pkg_to_install = ['ponysay','cowsay'] diff --git a/pydiploy/bottle.py b/pydiploy/bottle.py index 63ccb02..d3fe8ae 100644 --- a/pydiploy/bottle.py +++ b/pydiploy/bottle.py @@ -33,6 +33,10 @@ def application_packages(update=False): fabric.api.execute( pydiploy.require.system.install_extra_ppa, env.extra_ppa_to_install) + if 'extra_source_to_install' in env: + fabric.api.execute( + pydiploy.require.system.install_extra_source, + env.extra_source_to_install) if 'extra_pkg_to_install' in env: fabric.api.execute( pydiploy.require.system.install_extra_packages, diff --git a/pydiploy/django.py b/pydiploy/django.py index d484629..be40388 100644 --- a/pydiploy/django.py +++ b/pydiploy/django.py @@ -42,6 +42,9 @@ def application_packages(update=False): if env.has_key('extra_ppa_to_install'): fabric.api.execute( pydiploy.require.system.install_extra_ppa, env.extra_ppa_to_install) + if env.has_key('extra_source_to_install'): + fabric.api.execute( + pydiploy.require.system.install_extra_source, env.extra_source_to_install) if env.has_key('extra_pkg_to_install'): fabric.api.execute( pydiploy.require.system.install_extra_packages, env.extra_pkg_to_install) diff --git a/pydiploy/params.py b/pydiploy/params.py index c0728e2..9935a99 100644 --- a/pydiploy/params.py +++ b/pydiploy/params.py @@ -45,6 +45,7 @@ 'optional_params': {'excluded_files': "file(s) to exclude when deploying", 'extra_ppa_to_install': "extra ppa(s) to install on remote", + 'extra_source_to_install': "extra debian source(s) to install on remote", 'extra_pkg_to_install': "extra package(s) to install on remote", 'cfg_shared_files': "shared file(s) to deploy in shared dir", 'extra_goals': "extra goal(s) to add to deploy", @@ -106,6 +107,7 @@ 'optional_params': {'excluded_files': "file(s) to exclude when deploying", 'extra_ppa_to_install': "extra ppa(s) to install on remote", + 'extra_source_to_install': "extra debian source(s) to install on remote", 'extra_pkg_to_install': "extra package(s) to install on remote", 'cfg_shared_files': "shared file(s) to deploy in shared dir", 'extra_goals': "extra goal(s) to add to deploy", @@ -169,6 +171,7 @@ 'optional_params': {'excluded_files': "file(s) to exclude when deploying", 'extra_ppa_to_install': "extra ppa(s) to install on remote", + 'extra_source_to_install': "extra debian source(s) to install on remote", 'extra_pkg_to_install': "extra package(s) to install on remote", 'cfg_shared_files': "shared file(s) to deploy in shared dir", 'extra_goals': "extra goal(s) to add to deploy", diff --git a/pydiploy/require/system.py b/pydiploy/require/system.py index 386c880..3f5e6bb 100644 --- a/pydiploy/require/system.py +++ b/pydiploy/require/system.py @@ -125,7 +125,7 @@ def check_python3_install(version='python3', update=False): @do_verbose def install_extra_packages(pkg, update=False): """ - Install extra packages on remote server + Installs extra packages on remote server """ fabtools.require.deb.packages(pkg, update=update) @@ -135,7 +135,7 @@ def install_extra_packages(pkg, update=False): def install_extra_ppa(extra_ppa): """ - Install extra ppa source on remote server + Installs extra ppa source on remote server """ for ppa in extra_ppa: fabtools.require.deb.ppa(ppa) @@ -143,5 +143,15 @@ def install_extra_ppa(extra_ppa): @do_verbose def is_systemd(): - """ return True if systemd is used """ + """ Returns True if systemd is used """ return fabtools.files.is_dir("/run/systemd/system") + + +@do_verbose +def install_extra_source(extra_source): + """ + + Installs extra debian source on remote server + """ + for source in extra_source: + fabtools.require.deb.source(*source) diff --git a/pydiploy/simple.py b/pydiploy/simple.py index 3ef3406..dab2390 100644 --- a/pydiploy/simple.py +++ b/pydiploy/simple.py @@ -33,6 +33,10 @@ def application_packages(update=False): fabric.api.execute( pydiploy.require.system.install_extra_ppa, env.extra_ppa_to_install) + if 'extra_source_to_install' in env: + fabric.api.execute( + pydiploy.require.system.install_extra_source, + env.extra_source_to_install) if 'extra_pkg_to_install' in env: fabric.api.execute( pydiploy.require.system.install_extra_packages, diff --git a/tests/test_bottle.py b/tests/test_bottle.py index be54592..5d8ef62 100644 --- a/tests/test_bottle.py +++ b/tests/test_bottle.py @@ -72,6 +72,13 @@ def test_application_packages(self, api_execute, deb_packages): self.assertTrue( str(api_execute.call_args_list[17]).find('call( Date: Wed, 7 Sep 2016 14:15:25 +0200 Subject: [PATCH 11/44] Fix cleaning of local temp dir + tests --- pydiploy/require/releases_manager.py | 34 ++++++++++++++++------------ tests/test_releases_manager.py | 31 +++++++++++++++++-------- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/pydiploy/require/releases_manager.py b/pydiploy/require/releases_manager.py index 0962108..bdb7f42 100644 --- a/pydiploy/require/releases_manager.py +++ b/pydiploy/require/releases_manager.py @@ -79,13 +79,18 @@ def deploy_code(): fabric.api.require('tag', provided_by=['tag', 'head']) fabric.api.require('remote_project_dir', provided_by=env.goals) + archive_prefix = '%s-%s' % (env.application_name, env.tag.lower()) + tarball = pydiploy.require.git.archive(env.application_name, - prefix='%s-%s/' % (env.application_name, - env.tag.lower()), + prefix='%s/' % archive_prefix, specific_folder=env.remote_repo_specific_folder if "remote_repo_specific_folder" in env else "", tag=env.tag, remote=env.remote_repo_url) - with fabric.api.lcd('/tmp'): + + with fabric.api.lcd(env.local_tmp_dir): + # remove existing extracted dir from tarball + if os.path.exists('%s/%s' % (env.local_tmp_dir,archive_prefix)): + fabric.api.local('rm -rf %s' % archive_prefix) fabric.api.local('tar xvf %s' % os.path.basename(tarball)) if 'run_tests_command' in env and env.run_tests_command: @@ -109,9 +114,9 @@ def deploy_code(): env.remote_shared_path, os.path.basename(cfg_shared_file)), use_sudo=True) if cfg_present is None: - fabtools.files.upload_template('/tmp/%s-%s/%s' % ( - env.application_name, - env.tag.lower(), + fabtools.files.upload_template('%s/%s/%s' % ( + env.local_tmp_dir, + archive_prefix, cfg_shared_file ), os.path.join( @@ -128,10 +133,9 @@ def deploy_code(): 'releases_path': env.remote_releases_path, 'time': time()} fabric.contrib.project.rsync_project(env.remote_current_release, - '/tmp/%s-%s/' % ( - env.application_name, - env.tag.lower( - )), + '%s/%s/' % ( + env.local_tmp_dir, + archive_prefix), delete=True, extra_opts='--links --rsync-path="sudo -u %s rsync"' % env.remote_owner, exclude=exclude_files) @@ -144,10 +148,10 @@ def deploy_code(): fabric.api.execute(symlink) # set current directory with new release fabric.api.execute(set_current) + # remove git local git archive tarball - fabric.api.lcd('rm %s' % tarball) - # remove local temp dir unarchived directory - fabric.api.lcd('rm -rf /tmp/%s-%s' % (env.application_name, env.tag.lower())) + with fabric.api.lcd(env.local_tmp_dir): + fabric.api.local('rm %s' % os.path.basename(tarball)) @do_verbose @@ -200,10 +204,10 @@ def symlink(): @do_verbose def run_tests(): - # run local unit test + # Runs local unit test authorized_commands = ['tox'] if env.run_tests_command in authorized_commands: - with fabric.api.lcd('/tmp/%s-%s/' % (env.application_name, env.tag.lower())): + with fabric.api.lcd('%s/%s-%s/' % (env.local_tmp_dir, env.application_name, env.tag.lower())): fabric.api.local(env.run_tests_command) else: fabric.api.abort(fabric.colors.red("wrong test command. Currently, only tox is supported")) diff --git a/tests/test_releases_manager.py b/tests/test_releases_manager.py index e37f66c..19107d9 100644 --- a/tests/test_releases_manager.py +++ b/tests/test_releases_manager.py @@ -7,8 +7,8 @@ from fabric.api import env from mock import call, Mock, patch from pydiploy.require.releases_manager import (cleanup, deploy_code, - rollback_code, set_current, - setup, symlink) + rollback_code, run_tests, + set_current, setup, symlink) class ReleasesManagerCheck(TestCase): @@ -41,12 +41,12 @@ def setUp(self): env.remote_owner = "remote_owner" env.remote_project_dir = "remote_project_dir" env.remote_releases_path = "remote_releases_path" + env.remote_repo_specific_folder = "mysubfolder" env.remote_repo_url = "remote_repo_url" env.remote_shared_path = "remote_shared_path" env.root_package_name = "root_package_name" - env.tag = "mytag" env.run_tests_command = 'tox' - env.remote_repo_specific_folder = "mysubfolder" + env.tag = "mytag" def tearDown(self): env.clear() @@ -96,7 +96,8 @@ def test_cleanup(self, api_sudo): @patch('pydiploy.require.git.archive', return_value="myarchive") @patch('fabric.contrib.project.rsync_project', return_value=Mock()) @patch('fabtools.files.is_file', return_value=None) - def test_deploy_code(self, is_file, rsync_project, git_archive, upload_template, api_execute, api_sudo, api_lcd, api_require, api_local, api_prompt, tag_exist): + @patch('os.path.exists', return_value=False) + def test_deploy_code(self, path_exists, is_file, rsync_project, git_archive, upload_template, api_execute, api_sudo, api_lcd, api_require, api_local, api_prompt, tag_exist): api_lcd.return_value.__exit__ = Mock() api_lcd.return_value.__enter__ = Mock() @@ -131,8 +132,8 @@ def test_deploy_code(self, is_file, rsync_project, git_archive, upload_template, self.assertTrue(api_lcd.called) self.assertTrue(str(api_lcd.call_args_list[1]).find('/tmp/appliname-mytag/') > 0) - self.assertTrue(str(api_lcd.call_args_list[2]).find('rm myarchive') > 0 ) - self.assertTrue(str(api_lcd.call_args_list[3]).find('rm -rf /tmp/appliname-mytag') > 0 ) + #self.assertTrue(str(api_lcd.call_args_list[2]).find('rm myarchive') > 0 ) + #self.assertTrue(str(api_lcd.call_args_list[3]).find('rm -rf /tmp/appliname-mytag') > 0 ) self.assertTrue(api_local.called) self.assertTrue(str(api_local.call_args_list[0]).find('tar xvf myarchive')) self.assertTrue(str(api_local.call_args_list[1]).find(env.run_tests_command)) @@ -159,6 +160,10 @@ def test_deploy_code(self, is_file, rsync_project, git_archive, upload_template, self.assertTrue(api_prompt.called) self.assertEqual(env.tag, '4.0') + path_exists.return_value=True + deploy_code() + self.assertTrue(api_local.called) + @patch('fabric.api.puts', return_value=Mock()) @patch('fabric.api.sudo', return_value=Mock()) @patch('pydiploy.prepare.process_releases', return_value=Mock()) @@ -185,8 +190,6 @@ def test_rollback_code(self, process_rel, api_sudo, api_puts): call('rm remote_current_path; ln -s 3.0 remote_current_path && rm -rf remote_releases_path/4.0')) - - @patch('fabric.api.sudo', return_value=Mock()) def test_symlink(self, api_sudo): symlink() @@ -199,3 +202,13 @@ def test_symlink(self, api_sudo): symlink() self.assertTrue(str(api_sudo.call_args_list[4]).find( "ln -nfs remote_shared_path/symdir remote_current_release/symdir") > 0) + + @patch('fabric.api.abort', return_value=Mock()) + @patch('fabric.api.local', return_value=Mock()) + @patch('fabric.api.lcd', return_value=Mock()) + def test_run_tests(self, api_lcd, api_local, api_abort): + api_lcd.return_value.__exit__ = Mock() + api_lcd.return_value.__enter__ = Mock() + run_tests() + env.run_tests_command = 'dtc_les_tests' + run_tests() From 98bea7bd41fe84a173d5f4dc8d86610a5e1accac Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Thu, 8 Sep 2016 12:30:58 +0200 Subject: [PATCH 12/44] Remove ldap & pgsql pkg install for django... --- pydiploy/django.py | 4 ---- pydiploy/require/nginx.py | 5 +++-- pydiploy/require/system.py | 6 ++++-- tests/test_django.py | 16 ++++++---------- 4 files changed, 13 insertions(+), 18 deletions(-) diff --git a/pydiploy/django.py b/pydiploy/django.py index be40388..6b881ad 100644 --- a/pydiploy/django.py +++ b/pydiploy/django.py @@ -30,10 +30,6 @@ def wrap_deploy(): def application_packages(update=False): """ Installs all packages for django webapp """ fabtools.require.deb.packages(['gettext'], update=update) - # TODO contextual installation of ldap packages & postgres packages - # requirements !!! - fabric.api.execute(pydiploy.require.databases.ldap.ldap_pkg, use_sudo=True) - fabric.api.execute(pydiploy.require.databases.postgres.postgres_pkg) if env.remote_python_version >= 3: fabric.api.execute(pydiploy.require.system.check_python3_install, diff --git a/pydiploy/require/nginx.py b/pydiploy/require/nginx.py index fb1c2c7..eb9d8d4 100644 --- a/pydiploy/require/nginx.py +++ b/pydiploy/require/nginx.py @@ -7,7 +7,7 @@ import fabric import fabtools import pydiploy -from fabric.api import env +from fabric.api import env, warn_only from pydiploy.decorators import do_verbose from .system import is_systemd @@ -78,7 +78,8 @@ def nginx_started(): """ Returns true/false depending on nginx service is started """ if is_systemd(): # return fabtools.systemd.is_running('nginx') - return 'inactive' not in fabric.api.sudo('systemctl is-active nginx.service') + with warn_only(): + return 'inactive' not in fabric.api.sudo('systemctl is-active nginx.service') else: return fabtools.service.is_running('nginx') diff --git a/pydiploy/require/system.py b/pydiploy/require/system.py index 3f5e6bb..e936ae6 100644 --- a/pydiploy/require/system.py +++ b/pydiploy/require/system.py @@ -115,9 +115,11 @@ def check_python3_install(version='python3', update=False): """ if not package_installed(version): - if fabtools.system.distrib_id() == 'Ubuntu' and float(fabtools.system.distrib_release()) < 13.10: - fabtools.require.deb.packages(['python-software-properties'], + # TODO check for others ubuntu"s versions !!!!! + if fabtools.system.distrib_id() == 'Ubuntu' and float(fabtools.system.distrib_release()) < 13.10 or float(fabtools.system.distrib_release()) >= 16.04: + fabtools.require.deb.packages(['software-properties-common'], update=update) + # Install mighty PPA fabtools.require.deb.ppa('ppa:fkrull/deadsnakes') fabtools.require.deb.package(version, update=True) diff --git a/tests/test_django.py b/tests/test_django.py index a648fd4..772c417 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -47,39 +47,35 @@ def test_application_packages(self, api_execute, deb_packages): deb_packages.call_args, call(['gettext'], update=False)) self.assertTrue(api_execute.called) self.assertTrue( - str(api_execute.call_args_list[0]).find('call(= 3 env.remote_python_version = 3 application_packages() self.assertTrue(api_execute.called) self.assertTrue( - str(api_execute.call_args_list[5]).find('call( Date: Thu, 8 Sep 2016 12:39:22 +0200 Subject: [PATCH 13/44] Remove useless stuff (ldap+pgsl pkg) for bottle+simple... --- pydiploy/bottle.py | 3 --- pydiploy/simple.py | 3 --- tests/test_bottle.py | 14 +++++--------- tests/test_simple.py | 14 +++++--------- 4 files changed, 10 insertions(+), 24 deletions(-) diff --git a/pydiploy/bottle.py b/pydiploy/bottle.py index d3fe8ae..29c47bb 100644 --- a/pydiploy/bottle.py +++ b/pydiploy/bottle.py @@ -22,9 +22,6 @@ def wrap_deploy(): def application_packages(update=False): """ Installs all packages for the app """ fabtools.require.deb.packages(['gettext'], update=update) - # TODO contextual installation of ldap packages & postgres packages !!! - fabric.api.execute(pydiploy.require.databases.ldap.ldap_pkg, use_sudo=True) - fabric.api.execute(pydiploy.require.databases.postgres.postgres_pkg) if env.remote_python_version >= 3: fabric.api.execute(pydiploy.require.system.check_python3_install, version='python%s' % env.remote_python_version) diff --git a/pydiploy/simple.py b/pydiploy/simple.py index dab2390..9eeaa9c 100644 --- a/pydiploy/simple.py +++ b/pydiploy/simple.py @@ -22,9 +22,6 @@ def wrap_deploy(): def application_packages(update=False): """ Installs all packages for the app """ fabtools.require.deb.packages(['gettext'], update=update) - # TODO contextual installation of ldap packages & postgres packages !!! - fabric.api.execute(pydiploy.require.databases.ldap.ldap_pkg, use_sudo=True) - fabric.api.execute(pydiploy.require.databases.postgres.postgres_pkg) if env.remote_python_version >= 3: fabric.api.execute(pydiploy.require.system.check_python3_install, version='python%s' % env.remote_python_version) diff --git a/tests/test_bottle.py b/tests/test_bottle.py index 5d8ef62..a8bd1da 100644 --- a/tests/test_bottle.py +++ b/tests/test_bottle.py @@ -45,39 +45,35 @@ def test_application_packages(self, api_execute, deb_packages): deb_packages.call_args, call(['gettext'], update=False)) self.assertTrue(api_execute.called) self.assertTrue( - str(api_execute.call_args_list[0]).find('call(= 3 env.remote_python_version = 3 application_packages() self.assertTrue(api_execute.called) self.assertTrue( - str(api_execute.call_args_list[5]).find('call(= 3 env.remote_python_version = 3 application_packages() self.assertTrue(api_execute.called) self.assertTrue( - str(api_execute.call_args_list[5]).find('call( Date: Tue, 13 Sep 2016 15:35:30 +0200 Subject: [PATCH 14/44] New release 1.1.7.2 --- pydiploy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydiploy/version.py b/pydiploy/version.py index 33f8c25..538091c 100644 --- a/pydiploy/version.py +++ b/pydiploy/version.py @@ -1,3 +1,3 @@ # pydiploy version -__version__ = '1.1.7.1' +__version__ = '1.1.7.2' __version_info__ = tuple([ int(num) for num in __version__.split('.')]) From f2fb485e1ff73e8c3840e143254c242778fc5014 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Tue, 13 Sep 2016 16:05:04 +0200 Subject: [PATCH 15/44] Add extra_source_to_install in fabfile's examples --- .gitignore | 1 + pydiploy/examples/bottle_fabfile.py | 1 + pydiploy/examples/django_fabfile.py | 1 + pydiploy/examples/simple_fabfile.py | 1 + 4 files changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 43c5466..f8dfe05 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,4 @@ docs/_build .fab_tasks~ htmlcov .cache +.ropeproject/* diff --git a/pydiploy/examples/bottle_fabfile.py b/pydiploy/examples/bottle_fabfile.py index fadbf55..ca1184f 100644 --- a/pydiploy/examples/bottle_fabfile.py +++ b/pydiploy/examples/bottle_fabfile.py @@ -57,6 +57,7 @@ # env.excluded_files = ['pron.jpg'] # file(s) that rsync should exclude when deploying app # env.extra_ppa_to_install = ['ppa:vincent-c/ponysay'] # extra ppa source(s) to use # env.extra_pkg_to_install = ['ponysay'] # extra debian/ubuntu package(s) to install on remote +# env.extra_source_to_install = [['mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'], ['deb-src', 'http://site.example.com/debian', 'distribution', 'component1', 'component2', 'component3']] # env.cfg_shared_files = ['config','/app/path/to/config/config_file'] # config files to be placed in shared config dir # env.extra_symlink_dirs = ['mydir','/app/mydir'] # dirs to be symlinked in shared directory # env.extra_goals = ['preprod'] # add extra goal(s) to defaults (test,dev,prod) diff --git a/pydiploy/examples/django_fabfile.py b/pydiploy/examples/django_fabfile.py index e20ca98..3cb3dc0 100644 --- a/pydiploy/examples/django_fabfile.py +++ b/pydiploy/examples/django_fabfile.py @@ -57,6 +57,7 @@ # env.excluded_files = ['pron.jpg'] # file(s) that rsync should exclude when deploying app # env.extra_ppa_to_install = ['ppa:vincent-c/ponysay'] # extra ppa source(s) to use # env.extra_pkg_to_install = ['ponysay'] # extra debian/ubuntu package(s) to install on remote +# env.extra_source_to_install = [['mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'], ['deb-src', 'http://site.example.com/debian', 'distribution', 'component1', 'component2', 'component3']] # env.cfg_shared_files = ['config','/app/path/to/config/config_file'] # config files to be placed in shared config dir # env.extra_symlink_dirs = ['mydir','/app/mydir'] # dirs to be symlinked in shared directory # env.extra_goals = ['preprod'] # add extra goal(s) to defaults (test,dev,prod) diff --git a/pydiploy/examples/simple_fabfile.py b/pydiploy/examples/simple_fabfile.py index 642eceb..e4983c0 100644 --- a/pydiploy/examples/simple_fabfile.py +++ b/pydiploy/examples/simple_fabfile.py @@ -46,6 +46,7 @@ # env.excluded_files = ['pron.jpg'] # file(s) that rsync should exclude when deploying app # env.extra_ppa_to_install = ['ppa:vincent-c/ponysay'] # extra ppa source(s) to use # env.extra_pkg_to_install = ['ponysay'] # extra debian/ubuntu package(s) to install on remote +# env.extra_source_to_install = [['mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'], ['deb-src', 'http://site.example.com/debian', 'distribution', 'component1', 'component2', 'component3']] # env.cfg_shared_files = ['config','/app/path/to/config/config_file'] # config files to be placed in shared config dir # env.extra_symlink_dirs = ['mydir','/app/mydir'] # dirs to be symlinked in shared directory # env.extra_goals = ['preprod'] # add extra goal(s) to defaults (test,dev,prod) From 106c14111f2237c07092e6ef15edc75c636382b5 Mon Sep 17 00:00:00 2001 From: Morgan Date: Mon, 20 Mar 2017 15:21:34 +0100 Subject: [PATCH 16/44] remove ignored apps from the collectstatic command --- pydiploy/require/django/command.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pydiploy/require/django/command.py b/pydiploy/require/django/command.py index 574fa72..80e9665 100644 --- a/pydiploy/require/django/command.py +++ b/pydiploy/require/django/command.py @@ -37,9 +37,10 @@ def django_prepare(): with fabric.api.settings(warn_only=True): fabric.api.sudo('python manage.py migrate') fabric.api.sudo('python manage.py compilemessages') - ignore = ('rest_framework', 'django_extensions') - fabric.api.sudo('python manage.py collectstatic --noinput -i %s' % - ' -i '.join(ignore)) + # ignore = ('rest_framework', 'django_extensions') + # fabric.api.sudo('python manage.py collectstatic --noinput -i %s' % + # ' -i '.join(ignore)) + fabric.api.sudo('python manage.py collectstatic --noinput') fabric.api.get(os.path.join(env.remote_current_path, 'assets'), local_path=env.local_tmp_dir) @@ -83,6 +84,3 @@ def django_get_version(): with fabric.api.settings(sudo_user=env.remote_owner): django_version = fabric.api.sudo( 'python -c "import django;print(django.get_version())"') - - - From 28398f2bfe74c4cad93f70eec8b40569329b1ffb Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Fri, 13 Jan 2017 16:23:26 +0100 Subject: [PATCH 17/44] Fix cleanup method bug --- pydiploy/require/releases_manager.py | 1 + tests/test_releases_manager.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pydiploy/require/releases_manager.py b/pydiploy/require/releases_manager.py index bdb7f42..c66a156 100644 --- a/pydiploy/require/releases_manager.py +++ b/pydiploy/require/releases_manager.py @@ -44,6 +44,7 @@ def cleanup(): """ Cleans old stuff on remote server """ + fabric.api.execute(pydiploy.prepare.process_releases) if 'releases' in env and len(env.releases) >= env.keep_releases: directories = env.releases directories.reverse() diff --git a/tests/test_releases_manager.py b/tests/test_releases_manager.py index 19107d9..df16e79 100644 --- a/tests/test_releases_manager.py +++ b/tests/test_releases_manager.py @@ -79,7 +79,8 @@ def test_setup(self, api_execute, api_sudo): "mkdir -p remote_shared_path/symdir") > 0) @patch('fabric.api.sudo', return_value=Mock()) - def test_cleanup(self, api_sudo): + @patch('pydiploy.prepare.process_releases', return_value=Mock()) + def test_cleanup(self, rel_manager, api_sudo): cleanup() self.assertTrue(api_sudo.called) self.assertEqual( From 917ad0c76706022a178e1369d1d45600a956c5e6 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Fri, 13 Jan 2017 16:27:00 +0100 Subject: [PATCH 18/44] New release 1.1.7.3 --- pydiploy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydiploy/version.py b/pydiploy/version.py index 538091c..49e44b2 100644 --- a/pydiploy/version.py +++ b/pydiploy/version.py @@ -1,3 +1,3 @@ # pydiploy version -__version__ = '1.1.7.2' +__version__ = '1.1.7.3' __version_info__ = tuple([ int(num) for num in __version__.split('.')]) From 301e8951347e3958deab6448a8508f66365b684b Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Fri, 13 Jan 2017 16:39:46 +0100 Subject: [PATCH 19/44] Fix setup.py classifiers format --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2ab9707..96abc97 100644 --- a/setup.py +++ b/setup.py @@ -36,11 +36,11 @@ 'pydiploy_sublime_snippet = pydiploy.scripts:sublime_text_snippet', ] }, - classifiers=( + classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python :: 2.7' - ) + ] ) From ed6575646aa5b8b6766713be5c354932bb18144f Mon Sep 17 00:00:00 2001 From: Morgan Date: Mon, 20 Mar 2017 15:24:29 +0100 Subject: [PATCH 20/44] fix unit test for collectstatic --- tests/test_require_django.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_require_django.py b/tests/test_require_django.py index 66f73da..60115c5 100644 --- a/tests/test_require_django.py +++ b/tests/test_require_django.py @@ -85,7 +85,7 @@ def test_django_prepare(self, api_get, api_sudo, api_settings, api_cd, python_vi call('python manage.py syncdb --noinput'), call('python manage.py migrate'), call('python manage.py compilemessages'), - call('python manage.py collectstatic --noinput -i rest_framework -i django_extensions')]) + call('python manage.py collectstatic --noinput')]) @patch('fabtools.python.virtualenv', return_value=Mock()) @patch('fabric.api.cd', return_value=Mock()) @@ -297,4 +297,3 @@ def test_deploy_wsgi_file(self, upload_template): # str(upload_template.call_args).find("use_jinja=True") > 0) # self.assertTrue( # str(upload_template.call_args).find("user='owner'") > 0) - From 0c5be5006f0d5ac69b498d30ef0b7f3ea2725487 Mon Sep 17 00:00:00 2001 From: Morgan Date: Fri, 20 Oct 2017 15:42:23 +0200 Subject: [PATCH 21/44] fix test --- pydiploy/require/django/command.py | 7 ++++--- tests/test_require_django.py | 9 +++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pydiploy/require/django/command.py b/pydiploy/require/django/command.py index 80e9665..269d481 100644 --- a/pydiploy/require/django/command.py +++ b/pydiploy/require/django/command.py @@ -11,6 +11,7 @@ import fabtools from fabric.api import env from pydiploy.decorators import do_verbose +from distutils.version import LooseVersion @do_verbose @@ -27,10 +28,10 @@ def django_prepare(): with fabtools.python.virtualenv(env.remote_virtualenv_dir): with fabric.api.cd(env.remote_current_path): with fabric.api.settings(sudo_user=env.remote_owner): - django_version = fabric.api.sudo( - 'python -c "import django;print(django.get_version())"') + # django_version = fabric.api.sudo( + # 'python -c "import django;print(django.get_version())"') # check if django >= 1.8 no more syncdb migrate only ! - if django_version < "1.8": + if LooseVersion(django_get_version()) < LooseVersion("1.8"): fabric.api.sudo('python manage.py syncdb --noinput') # TODO add point in documentation # south needed with django < 1.7 !!!!! diff --git a/tests/test_require_django.py b/tests/test_require_django.py index 60115c5..7231875 100644 --- a/tests/test_require_django.py +++ b/tests/test_require_django.py @@ -45,7 +45,8 @@ def tearDown(self): @patch('fabric.api.settings', return_value=Mock()) @patch('fabric.api.sudo', return_value=Mock()) @patch('fabric.api.get', return_value=Mock()) - def test_django_prepare(self, api_get, api_sudo, api_settings, api_cd, python_virtualenv, api_local, api_lcd): + @patch('pydiploy.require.django.command.django_get_version', return_value="1.8") + def test_django_prepare(self, django_get_version, api_get, api_sudo, api_settings, api_cd, python_virtualenv, api_local, api_lcd): api_lcd.return_value.__exit__ = Mock() api_lcd.return_value.__enter__ = Mock() @@ -81,11 +82,11 @@ def test_django_prepare(self, api_get, api_sudo, api_settings, api_cd, python_vi self.assertTrue(api_sudo.called) self.assertEqual( api_sudo.call_args_list, [ - call('python -c "import django;print(django.get_version())"'), - call('python manage.py syncdb --noinput'), + # call('python -c "import django;print(django.get_version())"'), + # call('python manage.py syncdb --noinput'), call('python manage.py migrate'), call('python manage.py compilemessages'), - call('python manage.py collectstatic --noinput')]) + call('python manage.py collectstatic --noinput')]) @patch('fabtools.python.virtualenv', return_value=Mock()) @patch('fabric.api.cd', return_value=Mock()) From 94a006688a0cc8c993238f9dfedaa09458d82090 Mon Sep 17 00:00:00 2001 From: Morgan Date: Fri, 20 Oct 2017 15:45:18 +0200 Subject: [PATCH 22/44] add test 1.6 --- tests/test_require_django.py | 55 ++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_require_django.py b/tests/test_require_django.py index 7231875..dcf1d61 100644 --- a/tests/test_require_django.py +++ b/tests/test_require_django.py @@ -88,6 +88,61 @@ def test_django_prepare(self, django_get_version, api_get, api_sudo, api_setting call('python manage.py compilemessages'), call('python manage.py collectstatic --noinput')]) + + + @patch('fabric.api.lcd', return_value=Mock()) + @patch('fabric.api.local', return_value=Mock()) + @patch('fabtools.python.virtualenv', return_value=Mock()) + @patch('fabric.api.cd', return_value=Mock()) + @patch('fabric.api.settings', return_value=Mock()) + @patch('fabric.api.sudo', return_value=Mock()) + @patch('fabric.api.get', return_value=Mock()) + @patch('pydiploy.require.django.command.django_get_version', return_value="1.6") + def test_django_prepare(self, django_get_version, api_get, api_sudo, api_settings, api_cd, python_virtualenv, api_local, api_lcd): + + api_lcd.return_value.__exit__ = Mock() + api_lcd.return_value.__enter__ = Mock() + + python_virtualenv.return_value.__exit__ = Mock() + python_virtualenv.return_value.__enter__ = Mock() + + api_cd.return_value.__exit__ = Mock() + api_cd.return_value.__enter__ = Mock() + + api_settings.return_value.__exit__ = Mock() + api_settings.return_value.__enter__ = Mock() + + django_prepare() + + self.assertTrue(python_virtualenv.called) + self.assertEqual( + python_virtualenv.call_args, call('remote_virtualenv_dir')) + + self.assertTrue(api_cd.called) + self.assertEqual(api_cd.call_args, call('remote_current_path')) + + self.assertTrue(api_settings.called) + self.assertEqual(api_settings.call_args, call(warn_only=True)) + + self.assertTrue(api_settings.called) + self.assertEqual(api_settings.call_args, call(warn_only=True)) + + self.assertTrue(api_get.called) + self.assertEqual(api_get.call_args, call( + 'remote_current_path/assets', local_path='local_tmp_dir')) + + self.assertTrue(api_sudo.called) + self.assertEqual( + api_sudo.call_args_list, [ + # call('python -c "import django;print(django.get_version())"'), + call('python manage.py syncdb --noinput'), + call('python manage.py migrate'), + call('python manage.py compilemessages'), + call('python manage.py collectstatic --noinput')]) + + + + @patch('fabtools.python.virtualenv', return_value=Mock()) @patch('fabric.api.cd', return_value=Mock()) @patch('fabric.api.settings', return_value=Mock()) From 3f2744ab4829bce29405b92013982473777642df Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Mon, 23 Oct 2017 15:40:33 +0200 Subject: [PATCH 23/44] New release 1.1.7.4 --- pydiploy/version.py | 2 +- tools/pydiployfabfile.sublime-snippet | 2 +- tools/pydiployfabfilebottle.sublime-snippet | 2 +- tools/pydiployfabfilesimple.sublime-snippet | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pydiploy/version.py b/pydiploy/version.py index 49e44b2..72e9eaa 100644 --- a/pydiploy/version.py +++ b/pydiploy/version.py @@ -1,3 +1,3 @@ # pydiploy version -__version__ = '1.1.7.3' +__version__ = '1.1.7.4' __version_info__ = tuple([ int(num) for num in __version__.split('.')]) diff --git a/tools/pydiployfabfile.sublime-snippet b/tools/pydiployfabfile.sublime-snippet index 7736e05..7462190 100644 --- a/tools/pydiployfabfile.sublime-snippet +++ b/tools/pydiployfabfile.sublime-snippet @@ -302,7 +302,7 @@ def custom_manage_cmd(cmd): execute(pydiploy_custom_command,cmd) ]]> -Pydiploy fabfile 1.1.7.1 +Pydiploy fabfile 1.1.7.4 pydiployfab source.python diff --git a/tools/pydiployfabfilebottle.sublime-snippet b/tools/pydiployfabfilebottle.sublime-snippet index 43f5101..40349e4 100644 --- a/tools/pydiployfabfilebottle.sublime-snippet +++ b/tools/pydiployfabfilebottle.sublime-snippet @@ -293,7 +293,7 @@ def set_up(): execute(pydiploy_set_up) ]]> -Pydiploy fabfile bottle 1.1.7.1 +Pydiploy fabfile bottle 1.1.7.4 pydiployfabbottle source.python diff --git a/tools/pydiployfabfilesimple.sublime-snippet b/tools/pydiployfabfilesimple.sublime-snippet index 7cb5516..c320e71 100644 --- a/tools/pydiployfabfilesimple.sublime-snippet +++ b/tools/pydiployfabfilesimple.sublime-snippet @@ -162,7 +162,7 @@ def post_install_backend(): execute(pydiploy_postinstall_backend) ]]> -Pydiploy fabfile simple 1.1.7.1 +Pydiploy fabfile simple 1.1.7.4 pydiployfabsimple source.python From 75635d0375dc5c16b8348ab385cf0e2e5366679a Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Tue, 24 Oct 2017 10:00:18 +0200 Subject: [PATCH 24/44] Fix django_get_version returning nothing :) --- pydiploy/require/django/command.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pydiploy/require/django/command.py b/pydiploy/require/django/command.py index 269d481..712ed12 100644 --- a/pydiploy/require/django/command.py +++ b/pydiploy/require/django/command.py @@ -85,3 +85,5 @@ def django_get_version(): with fabric.api.settings(sudo_user=env.remote_owner): django_version = fabric.api.sudo( 'python -c "import django;print(django.get_version())"') + + return django_version From cab64cb38c280b822f07624d060522973681e2ba Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Tue, 24 Oct 2017 10:05:26 +0200 Subject: [PATCH 25/44] New release 1.1.7.5 --- .gitignore | 2 +- pydiploy/require/django/command.py | 4 ++++ pydiploy/version.py | 2 +- tools/pydiployfabfile.sublime-snippet | 2 +- tools/pydiployfabfilebottle.sublime-snippet | 2 +- tools/pydiployfabfilesimple.sublime-snippet | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index f8dfe05..dcbd41f 100644 --- a/.gitignore +++ b/.gitignore @@ -50,4 +50,4 @@ docs/_build .fab_tasks~ htmlcov .cache -.ropeproject/* +.ropeproject diff --git a/pydiploy/require/django/command.py b/pydiploy/require/django/command.py index 712ed12..69d3b4a 100644 --- a/pydiploy/require/django/command.py +++ b/pydiploy/require/django/command.py @@ -80,6 +80,10 @@ def django_custom_cmd(commands): @do_verbose def django_get_version(): """ Gets django version on remote """ + + # hopefully never compare with django_version=0 :) + django_version = 0 + with fabtools.python.virtualenv(env.remote_virtualenv_dir): with fabric.api.cd(env.remote_current_path): with fabric.api.settings(sudo_user=env.remote_owner): diff --git a/pydiploy/version.py b/pydiploy/version.py index 72e9eaa..add028a 100644 --- a/pydiploy/version.py +++ b/pydiploy/version.py @@ -1,3 +1,3 @@ # pydiploy version -__version__ = '1.1.7.4' +__version__ = '1.1.7.5' __version_info__ = tuple([ int(num) for num in __version__.split('.')]) diff --git a/tools/pydiployfabfile.sublime-snippet b/tools/pydiployfabfile.sublime-snippet index 7462190..6f661d1 100644 --- a/tools/pydiployfabfile.sublime-snippet +++ b/tools/pydiployfabfile.sublime-snippet @@ -302,7 +302,7 @@ def custom_manage_cmd(cmd): execute(pydiploy_custom_command,cmd) ]]> -Pydiploy fabfile 1.1.7.4 +Pydiploy fabfile 1.1.7.5 pydiployfab source.python diff --git a/tools/pydiployfabfilebottle.sublime-snippet b/tools/pydiployfabfilebottle.sublime-snippet index 40349e4..b0281f5 100644 --- a/tools/pydiployfabfilebottle.sublime-snippet +++ b/tools/pydiployfabfilebottle.sublime-snippet @@ -293,7 +293,7 @@ def set_up(): execute(pydiploy_set_up) ]]> -Pydiploy fabfile bottle 1.1.7.4 +Pydiploy fabfile bottle 1.1.7.5 pydiployfabbottle source.python diff --git a/tools/pydiployfabfilesimple.sublime-snippet b/tools/pydiployfabfilesimple.sublime-snippet index c320e71..b223269 100644 --- a/tools/pydiployfabfilesimple.sublime-snippet +++ b/tools/pydiployfabfilesimple.sublime-snippet @@ -162,7 +162,7 @@ def post_install_backend(): execute(pydiploy_postinstall_backend) ]]> -Pydiploy fabfile simple 1.1.7.4 +Pydiploy fabfile simple 1.1.7.5 pydiployfabsimple source.python From e9a6f60b07e4a1e2b0a1f1fd89497449373c8ff1 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Fri, 12 Jan 2018 10:16:40 +0100 Subject: [PATCH 26/44] Update new deadsnakes ppa url More info here : https://github.com/apache/incubator-mxnet/pull/9268 https://github.com/deadsnakes/issues/issues/54 --- pydiploy/require/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydiploy/require/system.py b/pydiploy/require/system.py index e936ae6..d1a205f 100644 --- a/pydiploy/require/system.py +++ b/pydiploy/require/system.py @@ -120,7 +120,7 @@ def check_python3_install(version='python3', update=False): fabtools.require.deb.packages(['software-properties-common'], update=update) # Install mighty PPA - fabtools.require.deb.ppa('ppa:fkrull/deadsnakes') + fabtools.require.deb.ppa('ppa:deadsnakes/ppa') fabtools.require.deb.package(version, update=True) From 3b2fe536ca17f2e617c40d7f460f130ed8486716 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Fri, 12 Jan 2018 14:15:05 +0100 Subject: [PATCH 27/44] New version 1.1.7.6 --- pydiploy/version.py | 2 +- tools/pydiployfabfile.sublime-snippet | 2 +- tools/pydiployfabfilebottle.sublime-snippet | 2 +- tools/pydiployfabfilesimple.sublime-snippet | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pydiploy/version.py b/pydiploy/version.py index add028a..2c50d0d 100644 --- a/pydiploy/version.py +++ b/pydiploy/version.py @@ -1,3 +1,3 @@ # pydiploy version -__version__ = '1.1.7.5' +__version__ = '1.1.7.6' __version_info__ = tuple([ int(num) for num in __version__.split('.')]) diff --git a/tools/pydiployfabfile.sublime-snippet b/tools/pydiployfabfile.sublime-snippet index 6f661d1..330ed10 100644 --- a/tools/pydiployfabfile.sublime-snippet +++ b/tools/pydiployfabfile.sublime-snippet @@ -302,7 +302,7 @@ def custom_manage_cmd(cmd): execute(pydiploy_custom_command,cmd) ]]> -Pydiploy fabfile 1.1.7.5 +Pydiploy fabfile 1.1.7.6 pydiployfab source.python diff --git a/tools/pydiployfabfilebottle.sublime-snippet b/tools/pydiployfabfilebottle.sublime-snippet index b0281f5..5688963 100644 --- a/tools/pydiployfabfilebottle.sublime-snippet +++ b/tools/pydiployfabfilebottle.sublime-snippet @@ -293,7 +293,7 @@ def set_up(): execute(pydiploy_set_up) ]]> -Pydiploy fabfile bottle 1.1.7.5 +Pydiploy fabfile bottle 1.1.7.6 pydiployfabbottle source.python diff --git a/tools/pydiployfabfilesimple.sublime-snippet b/tools/pydiployfabfilesimple.sublime-snippet index b223269..4703ba2 100644 --- a/tools/pydiployfabfilesimple.sublime-snippet +++ b/tools/pydiployfabfilesimple.sublime-snippet @@ -162,7 +162,7 @@ def post_install_backend(): execute(pydiploy_postinstall_backend) ]]> -Pydiploy fabfile simple 1.1.7.5 +Pydiploy fabfile simple 1.1.7.6 pydiployfabsimple source.python From 0f5f61a088a6de533a0c29794cf7bafed2e960cd Mon Sep 17 00:00:00 2001 From: tschlecht Date: Thu, 28 Jun 2018 11:33:32 +0200 Subject: [PATCH 28/44] Fix fabric version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3e9f0c2..d6c2532 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -fabric +fabric<2 fabtools jinja2 six From 85f7d288a27bfc773129c06bc0a0b77ee54efd06 Mon Sep 17 00:00:00 2001 From: tschlecht Date: Thu, 28 Jun 2018 11:38:52 +0200 Subject: [PATCH 29/44] New version 1.1.7.7 --- pydiploy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydiploy/version.py b/pydiploy/version.py index 2c50d0d..018637e 100644 --- a/pydiploy/version.py +++ b/pydiploy/version.py @@ -1,3 +1,3 @@ # pydiploy version -__version__ = '1.1.7.6' +__version__ = '1.1.7.7' __version_info__ = tuple([ int(num) for num in __version__.split('.')]) From 28792a29490d9904278013ec1d64fd325f91d3a7 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Mon, 8 Oct 2018 14:29:39 +0200 Subject: [PATCH 30/44] Release 1.1.7.8 --- pydiploy/version.py | 2 +- tools/pydiployfabfile.sublime-snippet | 6 +++--- tools/pydiployfabfilebottle.sublime-snippet | 6 +++--- tools/pydiployfabfilesimple.sublime-snippet | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pydiploy/version.py b/pydiploy/version.py index 018637e..3c67113 100644 --- a/pydiploy/version.py +++ b/pydiploy/version.py @@ -1,3 +1,3 @@ # pydiploy version -__version__ = '1.1.7.7' +__version__ = '1.1.7.8' __version_info__ = tuple([ int(num) for num in __version__.split('.')]) diff --git a/tools/pydiployfabfile.sublime-snippet b/tools/pydiployfabfile.sublime-snippet index 330ed10..1678f25 100644 --- a/tools/pydiployfabfile.sublime-snippet +++ b/tools/pydiployfabfile.sublime-snippet @@ -59,7 +59,7 @@ env.keep_releases = 2 # number of old releases to keep before cleaning # env.excluded_files = ['pron.jpg'] # file(s) that rsync should exclude when deploying app # env.extra_ppa_to_install = ['ppa:vincent-c/ponysay'] # extra ppa source(s) to use # env.extra_pkg_to_install = ['ponysay'] # extra debian/ubuntu package(s) to install on remote -# env.extra_source_to_install = [['mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'],] +# env.extra_source_to_install = [['mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'], ['deb-src', 'http://site.example.com/debian', 'distribution', 'component1', 'component2', 'component3']] # env.cfg_shared_files = ['config','/app/path/to/config/config_file'] # config files to be placed in shared config dir # env.extra_symlink_dirs = ['mydir','/app/mydir'] # dirs to be symlinked in shared directory # env.extra_goals = ['preprod'] # add extra goal(s) to defaults (test,dev,prod) @@ -302,7 +302,7 @@ def custom_manage_cmd(cmd): execute(pydiploy_custom_command,cmd) ]]> -Pydiploy fabfile 1.1.7.6 +Pydiploy fabfile 1.1.7.8 pydiployfab source.python - + \ No newline at end of file diff --git a/tools/pydiployfabfilebottle.sublime-snippet b/tools/pydiployfabfilebottle.sublime-snippet index 5688963..68993de 100644 --- a/tools/pydiployfabfilebottle.sublime-snippet +++ b/tools/pydiployfabfilebottle.sublime-snippet @@ -58,8 +58,8 @@ env.req_pydiploy_version = '1.1.6.0' # env.dest_path = '' # if not set using env_local_tmp_dir # env.excluded_files = ['pron.jpg'] # file(s) that rsync should exclude when deploying app # env.extra_ppa_to_install = ['ppa:vincent-c/ponysay'] # extra ppa source(s) to use -# env.extra_source_to_install = [['mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'],] # env.extra_pkg_to_install = ['ponysay'] # extra debian/ubuntu package(s) to install on remote +# env.extra_source_to_install = [['mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'], ['deb-src', 'http://site.example.com/debian', 'distribution', 'component1', 'component2', 'component3']] # env.cfg_shared_files = ['config','/app/path/to/config/config_file'] # config files to be placed in shared config dir # env.extra_symlink_dirs = ['mydir','/app/mydir'] # dirs to be symlinked in shared directory # env.extra_goals = ['preprod'] # add extra goal(s) to defaults (test,dev,prod) @@ -293,7 +293,7 @@ def set_up(): execute(pydiploy_set_up) ]]> -Pydiploy fabfile bottle 1.1.7.6 +Pydiploy fabfile bottle 1.1.7.8 pydiployfabbottle source.python - + \ No newline at end of file diff --git a/tools/pydiployfabfilesimple.sublime-snippet b/tools/pydiployfabfilesimple.sublime-snippet index 4703ba2..a3f30bf 100644 --- a/tools/pydiployfabfilesimple.sublime-snippet +++ b/tools/pydiployfabfilesimple.sublime-snippet @@ -47,8 +47,8 @@ env.application_type = 'simple' # specify another type of application # env.dest_path = '' # if not set using env_local_tmp_dir # env.excluded_files = ['pron.jpg'] # file(s) that rsync should exclude when deploying app # env.extra_ppa_to_install = ['ppa:vincent-c/ponysay'] # extra ppa source(s) to use -# env.extra_source_to_install = [['mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'],] # env.extra_pkg_to_install = ['ponysay'] # extra debian/ubuntu package(s) to install on remote +# env.extra_source_to_install = [['mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'], ['deb-src', 'http://site.example.com/debian', 'distribution', 'component1', 'component2', 'component3']] # env.cfg_shared_files = ['config','/app/path/to/config/config_file'] # config files to be placed in shared config dir # env.extra_symlink_dirs = ['mydir','/app/mydir'] # dirs to be symlinked in shared directory # env.extra_goals = ['preprod'] # add extra goal(s) to defaults (test,dev,prod) @@ -162,7 +162,7 @@ def post_install_backend(): execute(pydiploy_postinstall_backend) ]]> -Pydiploy fabfile simple 1.1.7.6 +Pydiploy fabfile simple 1.1.7.8 pydiployfabsimple source.python - + \ No newline at end of file From 98451cf4eebd7466b608a075812f05c3439e41c8 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Fri, 19 Oct 2018 13:11:53 +0200 Subject: [PATCH 31/44] Fix problem when upgrading pip with sudo --- pydiploy/require/python/utils.py | 26 ++++++++++++++------------ pydiploy/require/python/virtualenv.py | 9 +-------- pydiploy/require/system.py | 9 ++++++++- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/pydiploy/require/python/utils.py b/pydiploy/require/python/utils.py index b652dcf..346b821 100644 --- a/pydiploy/require/python/utils.py +++ b/pydiploy/require/python/utils.py @@ -8,6 +8,7 @@ import fabtools from fabric.api import env from pydiploy.decorators import do_verbose +from pydiploy.require.system import shell @do_verbose @@ -41,22 +42,23 @@ def application_dependencies(upgrade_pkg, staging=True): env.remote_home, oracle_root_path) pip_cmd = 'ORACLE_HOME=%s pip' % oracle_full_path - # upgrade pip to latest version - fabtools.require.python.install('pip', - upgrade=True, - use_sudo=True, - user=env.remote_owner, - pip_cmd=pip_cmd, - quiet=True) + with shell("HOME=~%s %s" % (env.remote_owner, env.shell)): + # upgrade pip to latest version + fabtools.require.python.install('pip', + upgrade=True, + use_sudo=True, + user=env.remote_owner, + pip_cmd=pip_cmd, + quiet=True) - fabtools.python.install_requirements(requirements_file, + fabtools.python.install_requirements(requirements_file, use_sudo=True, user=env.remote_owner, upgrade=upgrade_pkg, pip_cmd='%s --no-cache-dir' % pip_cmd, quiet=True) - fabric.api.sudo( - 'pip install --log-file %s --quiet -e .' % pip_log , - user=env.remote_owner, - pty=False) + fabric.api.sudo( + 'pip install --log-file %s --quiet -e .' % pip_log , + user=env.remote_owner, + pty=False) diff --git a/pydiploy/require/python/virtualenv.py b/pydiploy/require/python/virtualenv.py index ed730b7..2f73b08 100644 --- a/pydiploy/require/python/virtualenv.py +++ b/pydiploy/require/python/virtualenv.py @@ -1,17 +1,10 @@ # -*- coding: utf-8 -*- """ This module is used for virtualenv relatives """ -from contextlib import contextmanager import fabtools from fabric.api import env from pydiploy.decorators import do_verbose - - -@contextmanager -def shell(new_shell): - old_shell, env.shell = env.shell, new_shell - yield - env.shell = old_shell +from pydiploy.require.system import shell @do_verbose diff --git a/pydiploy/require/system.py b/pydiploy/require/system.py index d1a205f..add2d99 100644 --- a/pydiploy/require/system.py +++ b/pydiploy/require/system.py @@ -5,7 +5,7 @@ """ - +from contextlib import contextmanager import fabric import fabtools from fabric.api import env @@ -157,3 +157,10 @@ def install_extra_source(extra_source): """ for source in extra_source: fabtools.require.deb.source(*source) + + +@contextmanager +def shell(new_shell): + old_shell, env.shell = env.shell, new_shell + yield + env.shell = old_shell From 6bfc9707c2a9d329377f51c843de2b2e0b6dda76 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Fri, 19 Oct 2018 13:13:15 +0200 Subject: [PATCH 32/44] Release 1.1.7.9 --- pydiploy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydiploy/version.py b/pydiploy/version.py index 3c67113..85aa75d 100644 --- a/pydiploy/version.py +++ b/pydiploy/version.py @@ -1,3 +1,3 @@ # pydiploy version -__version__ = '1.1.7.8' +__version__ = '1.1.7.9' __version_info__ = tuple([ int(num) for num in __version__.split('.')]) From f5774b649cfb071b545c4c5893852e4e9833bf63 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Fri, 7 Jun 2019 11:22:27 +0200 Subject: [PATCH 33/44] Fix units tests and fix dependency for ubuntu >= 18 --- pydiploy/require/circus.py | 28 ++++++++----- tests/test_require_circus.py | 81 ++++++++++++++++++++++++++++++++---- 2 files changed, 91 insertions(+), 18 deletions(-) diff --git a/pydiploy/require/circus.py b/pydiploy/require/circus.py index 3520c40..a8d4bc5 100644 --- a/pydiploy/require/circus.py +++ b/pydiploy/require/circus.py @@ -26,17 +26,23 @@ def circus_pkg(update=False): """ Installs packages relatives to circus """ - # install ubuntu ppa for libzmq-dev if ubuntu <= 10.04 - if fabtools.system.distrib_id() == 'Ubuntu' and fabtools.system.distrib_release() == '10.04': - fabtools.require.deb.packages(['python-software-properties'], - update=update) - fabtools.require.deb.ppa('ppa:chris-lea/zeromq') - fabtools.require.deb.ppa('ppa:chris-lea/libpgm') - - fabtools.require.deb.packages([ - 'libzmq-dev', - 'libevent-dev'], update=update) - # not used anymore installed in venv ! + # TOOLD ! + # install ubuntu ppa for libzmq-dev if ubuntu <= 10.04 + #if fabtools.system.distrib_id() == 'Ubuntu' and fabtools.system.distrib_release() == '10.04': + # fabtools.require.deb.packages(['python-software-properties'], + # fabtools.require.deb.ppa('ppa:chris-lea/zeromq') + # fabtools.require.deb.ppa('ppa:chris-lea/libpgm') + + if fabtools.system.distrib_id() == 'Ubuntu' and fabtools.system.distrib_release() >= '18.04': + fabtools.require.deb.packages([ + 'libzmq3-dev', + 'libevent-dev'], update=update) + + if fabtools.system.distrib_id() == 'Ubuntu' and fabtools.system.distrib_release() < '18.04': + fabtools.require.deb.packages([ + 'libzmq-dev', + 'libevent-dev'], update=update) + fabtools.require.python.install(env.get('circus_package_name', 'circus'), use_sudo=True, upgrade=update) diff --git a/tests/test_require_circus.py b/tests/test_require_circus.py index 74241da..ca87203 100644 --- a/tests/test_require_circus.py +++ b/tests/test_require_circus.py @@ -29,7 +29,76 @@ def tearDown(self): env.update(self.previous_env) @patch('fabtools.system.distrib_id', return_value='Ubuntu') - @patch('fabtools.system.distrib_release', return_value='10.04') + @patch('fabtools.system.distrib_release', return_value='16.04') + @patch('fabtools.require.deb.packages', return_value=Mock()) + @patch('fabtools.require.deb.ppa', return_value=Mock()) + @patch('fabtools.require.python.install', return_value=Mock()) + @patch('fabtools.files.upload_template', return_value=Mock()) + @patch('fabtools.require.files.directory', return_value=Mock()) + def test_circus_pkg_old(self, files_directory, upload_template, python_install, deb_ppa, deb_packages, distrib_release, distrib_id): + + # no_circus_web = True + env.no_circus_web = True + circus_pkg() + + self.assertTrue(distrib_release.called) + self.assertTrue(distrib_id.called) + + self.assertTrue(deb_packages.called) + self.assertEqual( + deb_packages.call_args_list, [call(['libzmq-dev', 'libevent-dev'], update=False)]) + + # self.assertTrue(deb_ppa.called) + # self.assertEqual(deb_ppa.call_args_list, [ + # call('ppa:chris-lea/zeromq'), call('ppa:chris-lea/libpgm')]) + + self.assertTrue(python_install.called) + + self.assertEqual( + python_install.call_args_list, [call('circus', upgrade=False, use_sudo=True)]) + + self.assertTrue(upload_template.called) + self.assertTrue( + str(upload_template.call_args).find("'circus.ini.tpl'") > 0) + self.assertTrue( + str(upload_template.call_args).find("'remote_home/.circus.ini'") > 0) + self.assertTrue(str(upload_template.call_args) + .find("template_dir='lib_path/templates'") > 0) + self.assertTrue( + str(upload_template.call_args).find("user='remote_owner'") > 0) + + self.assertTrue(files_directory.called) + self.assertEqual(files_directory.call_args, + call(owner='remote_owner', path='remote_home/.circus.d', use_sudo=True, group='remote_group', mode='750')) + + # test circus_package_name + env.circus_package_name = "https://github.com/githubaccount/circus/archive/master.zip" + circus_pkg() + self.assertEqual( + python_install.call_args_list[1], call( + 'https://github.com/githubaccount/circus/archive/master.zip', upgrade=False, use_sudo=True) + ) + + # test no_circus_web + del env['no_circus_web'] + del env['circus_package_name'] + circus_pkg() + self.assertEqual( + python_install.call_args_list[-3:], [call('circus', use_sudo=True, upgrade=False), + call('circus-web', use_sudo=True, upgrade=False), + call('gevent', use_sudo=True, upgrade=False)]) + + # test circus_backend + env.circus_backend = "test_circus_backend" + circus_pkg() + self.assertEqual( + python_install.call_args_list[-4:], [call('circus', use_sudo=True, upgrade=False), + call('circus-web', use_sudo=True, upgrade=False), + call('gevent', use_sudo=True, upgrade=False), + call('test_circus_backend', use_sudo=True, upgrade=False)]) + + @patch('fabtools.system.distrib_id', return_value='Ubuntu') + @patch('fabtools.system.distrib_release', return_value='18.04') @patch('fabtools.require.deb.packages', return_value=Mock()) @patch('fabtools.require.deb.ppa', return_value=Mock()) @patch('fabtools.require.python.install', return_value=Mock()) @@ -46,13 +115,11 @@ def test_circus_pkg(self, files_directory, upload_template, python_install, deb_ self.assertTrue(deb_packages.called) self.assertEqual( - deb_packages.call_args_list, [call( - ['python-software-properties'], update=False), - call(['libzmq-dev', 'libevent-dev'], update=False)]) + deb_packages.call_args_list, [call(['libzmq3-dev', 'libevent-dev'], update=False)]) - self.assertTrue(deb_ppa.called) - self.assertEqual(deb_ppa.call_args_list, [ - call('ppa:chris-lea/zeromq'), call('ppa:chris-lea/libpgm')]) + # self.assertTrue(deb_ppa.called) + # self.assertEqual(deb_ppa.call_args_list, [ + # call('ppa:chris-lea/zeromq'), call('ppa:chris-lea/libpgm')]) self.assertTrue(python_install.called) From 70e4978332880139650c0d0c1a214b20952d5656 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Fri, 7 Jun 2019 15:26:07 +0200 Subject: [PATCH 34/44] Enhance systemd support and add err. msg when repo url is wrong --- pydiploy/require/circus.py | 9 ++++++--- pydiploy/require/django/command.py | 5 +++-- pydiploy/require/releases_manager.py | 22 ++++++++++++++-------- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/pydiploy/require/circus.py b/pydiploy/require/circus.py index a8d4bc5..4f66c9b 100644 --- a/pydiploy/require/circus.py +++ b/pydiploy/require/circus.py @@ -17,7 +17,7 @@ import fabric import fabtools -from fabric.api import env, warn_only +from fabric.api import env, warn_only, hide from pydiploy.decorators import do_verbose from .system import is_systemd @@ -138,7 +138,8 @@ def app_reload(): start_cmd = 'systemctl start circus.service' status_cmd = 'systemctl is-active circus.service' with warn_only(): - running = 'inactive' not in fabric.api.sudo(status_cmd) + with hide(): + running = 'inactive' not in fabric.api.sudo(status_cmd) # Upstart else: start_cmd = 'start circus' @@ -150,4 +151,6 @@ def app_reload(): else: with fabric.api.settings(sudo_user=env.remote_owner): fabric.api.sudo('circusctl reloadconfig') - fabric.api.sudo('circusctl restart %s' % env.application_name) + with warn_only(): + with hide(): + fabric.api.sudo('circusctl restart %s' % env.application_name) diff --git a/pydiploy/require/django/command.py b/pydiploy/require/django/command.py index 69d3b4a..65b1a76 100644 --- a/pydiploy/require/django/command.py +++ b/pydiploy/require/django/command.py @@ -9,7 +9,7 @@ import fabric import fabtools -from fabric.api import env +from fabric.api import env, hide from pydiploy.decorators import do_verbose from distutils.version import LooseVersion @@ -37,7 +37,8 @@ def django_prepare(): # south needed with django < 1.7 !!!!! with fabric.api.settings(warn_only=True): fabric.api.sudo('python manage.py migrate') - fabric.api.sudo('python manage.py compilemessages') + with hide('warnings'): + fabric.api.sudo('python manage.py compilemessages') # ignore = ('rest_framework', 'django_extensions') # fabric.api.sudo('python manage.py collectstatic --noinput -i %s' % # ' -i '.join(ignore)) diff --git a/pydiploy/require/releases_manager.py b/pydiploy/require/releases_manager.py index c66a156..0f47f25 100644 --- a/pydiploy/require/releases_manager.py +++ b/pydiploy/require/releases_manager.py @@ -6,7 +6,8 @@ import fabric import fabtools import pydiploy -from fabric.api import env + +from fabric.api import env, warn_only from pydiploy.decorators import do_verbose @@ -133,13 +134,18 @@ def deploy_code(): env.remote_current_release = "%(releases_path)s/%(time).0f" % { 'releases_path': env.remote_releases_path, 'time': time()} - fabric.contrib.project.rsync_project(env.remote_current_release, - '%s/%s/' % ( - env.local_tmp_dir, - archive_prefix), - delete=True, - extra_opts='--links --rsync-path="sudo -u %s rsync"' % env.remote_owner, - exclude=exclude_files) + with warn_only(): + result = fabric.contrib.project.rsync_project(env.remote_current_release, + '%s/%s/' % ( + env.local_tmp_dir, + archive_prefix), + delete=True, + extra_opts='--links --rsync-path="sudo -u %s rsync"' % env.remote_owner, + exclude=exclude_files) + + # Wrong repository url (git archive is empty) + if result.return_code == 23: + fabric.api.abort(fabric.colors.red("Unable to use repository, please check repository url !")) fabric.api.sudo( 'chown -R %(user)s:%(group)s %(project_dir)s' % {'user': env.remote_owner, From aded2f6d108216f9d47aec0750a687fb2d356dd9 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Fri, 7 Jun 2019 15:58:07 +0200 Subject: [PATCH 35/44] Chaining context managers is better more to come --- pydiploy/require/circus.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pydiploy/require/circus.py b/pydiploy/require/circus.py index 4f66c9b..48e5df4 100644 --- a/pydiploy/require/circus.py +++ b/pydiploy/require/circus.py @@ -137,9 +137,8 @@ def app_reload(): if is_systemd(): start_cmd = 'systemctl start circus.service' status_cmd = 'systemctl is-active circus.service' - with warn_only(): - with hide(): - running = 'inactive' not in fabric.api.sudo(status_cmd) + with warn_only(), hide(): + running = 'inactive' not in fabric.api.sudo(status_cmd) # Upstart else: start_cmd = 'start circus' @@ -151,6 +150,5 @@ def app_reload(): else: with fabric.api.settings(sudo_user=env.remote_owner): fabric.api.sudo('circusctl reloadconfig') - with warn_only(): - with hide(): - fabric.api.sudo('circusctl restart %s' % env.application_name) + with warn_only(), hide(): + fabric.api.sudo('circusctl restart %s' % env.application_name) From 5d6d1eb78b56ff9dec139817f82747eea11db8a0 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Wed, 3 Jun 2020 14:12:07 +0200 Subject: [PATCH 36/44] WIP for now --- pydiploy/prepare.py | 119 ++++++++++++++++--------------- pydiploy/require/python/utils.py | 69 ++++++++++-------- pydiploy/require/system.py | 47 ++++++------ 3 files changed, 123 insertions(+), 112 deletions(-) diff --git a/pydiploy/prepare.py b/pydiploy/prepare.py index 381b5ec..7389192 100644 --- a/pydiploy/prepare.py +++ b/pydiploy/prepare.py @@ -18,6 +18,7 @@ import fabtools from fabric.api import env from pkg_resources import Requirement, resource_filename + from pydiploy.params import PARAMS from pydiploy.require.git import check_tag_exist from pydiploy.version import __version__, __version_info__ @@ -28,23 +29,23 @@ def tag(version): """ Defines tag to deploy """ if "pydiploy_version" in env: - fabric.api.abort(fabric.colors.red( - "tag should be set before calling goal (ex: fab tag:master test deploy)")) + fabric.api.abort(fabric.colors.red("tag should be set before calling goal (ex: fab tag:master test deploy)")) if check_tag_exist(version): env.tag = version else: - fabric.api.abort(fabric.colors.red( - "tag/branch provided is not in the repository please fix this first")) + fabric.api.abort(fabric.colors.red("tag/branch provided is not in the repository please fix this first")) def init_params(application_type='default'): """ Sets required params and its description """ try: - required_params, optional_params = PARAMS[application_type]['required_params'], PARAMS[application_type]['optional_params'] + required_params, optional_params = ( + PARAMS[application_type]['required_params'], + PARAMS[application_type]['optional_params'], + ) except KeyError: - fabric.api.abort(fabric.colors.red( - "application_type '%s' doesn't exists" % application_type)) + fabric.api.abort(fabric.colors.red("application_type '%s' doesn't exists" % application_type)) return required_params, optional_params @@ -57,12 +58,13 @@ def build_env(): if "req_pydiploy_version" in env: if not check_req_pydiploy_version(): if not fabric.contrib.console.confirm( - "\nYour fabfile require pydiploy %s and pydiploy %s is installed ! \ + "\nYour fabfile require pydiploy %s and pydiploy %s is installed ! \ \nBe sure that your fabfile complies last pydiploy evolutions. \ \nContinue at your own risks ! \ - \n\nDo you want to continue?" % - (fabric.colors.red(env.req_pydiploy_version), - fabric.colors.red(__version__)), default=False): + \n\nDo you want to continue?" + % (fabric.colors.red(env.req_pydiploy_version), fabric.colors.red(__version__)), + default=False, + ): fabric.api.abort("Aborting at user request.") # remote home cannot be empty or / path @@ -79,20 +81,15 @@ def build_env(): env.remote_project_dir = os.path.join(env.remote_home, env.application_name) if "tag" in env: - env.local_tmp_root_app = os.path.join(env.local_tmp_dir, - '%(application_name)s-%(tag)s' % env) - env.local_tmp_root_app_package = os.path.join(env.local_tmp_root_app, - env.root_package_name) + env.local_tmp_root_app = os.path.join(env.local_tmp_dir, '%(application_name)s-%(tag)s' % env) + env.local_tmp_root_app_package = os.path.join(env.local_tmp_root_app, env.root_package_name) env.remote_current_path = os.path.join(env.remote_project_dir, 'current') env.remote_releases_path = os.path.join(env.remote_project_dir, 'releases') env.remote_shared_path = os.path.join(env.remote_project_dir, 'shared') - env.remote_base_package_dir = os.path.join(env.remote_current_path, - env.root_package_name) - env.remote_settings_dir = os.path.join( - env.remote_base_package_dir, 'settings') - env.remote_settings_file = os.path.join(env.remote_settings_dir, - '%s.py' % env.goal) + env.remote_base_package_dir = os.path.join(env.remote_current_path, env.root_package_name) + env.remote_settings_dir = os.path.join(env.remote_base_package_dir, 'settings') + env.remote_settings_file = os.path.join(env.remote_settings_dir, '%s.py' % env.goal) env.lib_path = os.path.dirname(__file__) if "previous_settings_file" not in env: @@ -121,7 +118,9 @@ def build_env(): application_type = 'default' if not test_config(verbose=verbose_value, application_type=application_type): - if not fabric.contrib.console.confirm("Configuration test %s! Do you want to continue?" % fabric.colors.red('failed'), default=False): + if not fabric.contrib.console.confirm( + "Configuration test %s! Do you want to continue?" % fabric.colors.red('failed'), default=False + ): fabric.api.abort("Aborting at user request.") @@ -143,15 +142,15 @@ def test_config(verbose=True, application_type='default'): max_opt_desc_length = max(map(len, opt_params.values())) current_role = _get_current_role() - fabric.api.puts("\n\nConfiguration checking for role : %s, host : %s" % ( - fabric.colors.green(current_role), fabric.colors.green(env.host))) + fabric.api.puts( + "\n\nConfiguration checking for role : %s, host : %s" + % (fabric.colors.green(current_role), fabric.colors.green(env.host)) + ) for param, desc in sorted(req_params.items()): if param not in env: - err.append("%s -> %s : missing" % - (param.ljust(max_req_param_length), desc.ljust(max_req_desc_length))) + err.append("%s -> %s : missing" % (param.ljust(max_req_param_length), desc.ljust(max_req_desc_length))) elif not bool(env[param]): - err.append("%s -> %s : not set" % - (param.ljust(max_req_param_length), desc.ljust(max_req_desc_length))) + err.append("%s -> %s : not set" % (param.ljust(max_req_param_length), desc.ljust(max_req_desc_length))) elif verbose: req_parameters.append((param, env[param], desc)) @@ -164,31 +163,32 @@ def test_config(verbose=True, application_type='default'): if err: err_nb = len(err) if err_nb == len(req_params): - fabric.api.puts( - 'You need to configure correctly the fabfile please RTFM first !') + fabric.api.puts('You need to configure correctly the fabfile please RTFM first !') else: - fabric.api.puts('Config test failed (%s error%s) for role %s:' % - (err_nb, 's' if err_nb > 1 else '', current_role)) - fabric.api.puts('%s\n\n* %s\n' % ('-' * 30, '\n* '.join(err))) fabric.api.puts( - 'Please fix them or continue with possible errors.') + 'Config test failed (%s error%s) for role %s:' % (err_nb, 's' if err_nb > 1 else '', current_role) + ) + fabric.api.puts('%s\n\n* %s\n' % ('-' * 30, '\n* '.join(err))) + fabric.api.puts('Please fix them or continue with possible errors.') return False elif verbose: fabric.api.puts('\n\nRequired parameters list : \n\n') for param, value, description in req_parameters: - fabric.api.puts('* %s %s' % - (param.ljust(max_req_param_length), fabric.colors.green(value))) + fabric.api.puts('* %s %s' % (param.ljust(max_req_param_length), fabric.colors.green(value))) fabric.api.puts('\n\nOptional parameters list : \n\n') if len(opt_parameters): for param, value, description in opt_parameters: - value = fabric.colors.red("Warning initialized but not set") if not bool( - str(value)) ^ bool(value == None) else fabric.colors.green(value) - fabric.api.puts('* %s %s' % - (param.ljust(max_opt_param_length), value)) + value = ( + fabric.colors.red("Warning initialized but not set") + if not bool(str(value)) ^ bool(value == None) + else fabric.colors.green(value) + ) + fabric.api.puts('* %s %s' % (param.ljust(max_opt_param_length), value)) else: fabric.api.puts("No optional parameter found") - fabric.api.puts('\n\nRole : %s -> configuration %s!\n\n' % - (fabric.colors.green(current_role), fabric.colors.green("OK"))) + fabric.api.puts( + '\n\nRole : %s -> configuration %s!\n\n' % (fabric.colors.green(current_role), fabric.colors.green("OK")) + ) return True @@ -209,9 +209,8 @@ def check_req_pydiploy_version(): if "req_pydiploy_version" in env: major_version_installed = __version_info__[0:2] - major_version_required = tuple( - [int(num) for num in env.req_pydiploy_version.split('.')]) - if (major_version_installed != major_version_required[0:2]): + major_version_required = tuple([int(num) for num in env.req_pydiploy_version.split('.')]) + if major_version_installed != major_version_required[0:2]: return False return True @@ -244,23 +243,25 @@ def generate_fabfile_bottle(): def process_releases(): """ Populates env vars for releases (current, old...) """ if fabtools.files.is_dir(env.remote_releases_path): - env.releases = sorted(fabric.api.run('ls -x %(releases_path)s' % - {'releases_path': env.remote_releases_path}).split()) + env.releases = sorted( + fabric.api.run('ls -x %(releases_path)s' % {'releases_path': env.remote_releases_path}).split() + ) if len(env.releases) >= 1: env.current_revision = env.releases[-1] - env.current_release = "%(releases_path)s/%(current_revision)s" % \ - {'releases_path': env.remote_releases_path, - 'current_revision': env.current_revision} + env.current_release = "%(releases_path)s/%(current_revision)s" % { + 'releases_path': env.remote_releases_path, + 'current_revision': env.current_revision, + } # warning previous settings file before deployement !!!!! - env.previous_release_base_package_dir = os.path.join( - env.current_release, env.root_package_name) - env.previous_release_settings_dir = os.path.join( - env.previous_release_base_package_dir, 'settings') - env.previous_settings_file = os.path.join( - env.previous_release_settings_dir, '%s.py' % env.goal) + env.previous_release_base_package_dir = os.path.join(env.current_release, env.root_package_name) + env.previous_release_settings_dir = os.path.join(env.previous_release_base_package_dir, 'settings') + env.previous_settings_file = os.path.join(env.previous_release_settings_dir, '%s.py' % env.goal) + fabric.api.puts("settings :", env.previous_settings_file) if len(env.releases) > 1: env.previous_revision = env.releases[-2] - env.previous_release = "%(releases_path)s/%(previous_revision)s" % \ - {'releases_path': env.remote_releases_path, - 'previous_revision': env.previous_revision} + env.previous_release = "%(releases_path)s/%(previous_revision)s" % { + 'releases_path': env.remote_releases_path, + 'previous_revision': env.previous_revision, + } + diff --git a/pydiploy/require/python/utils.py b/pydiploy/require/python/utils.py index 346b821..65cdd10 100644 --- a/pydiploy/require/python/utils.py +++ b/pydiploy/require/python/utils.py @@ -15,10 +15,31 @@ def python_pkg(update=False): """ Installs python packages and pip """ - fabtools.require.deb.packages([ - '%s' % 'python-dev' if env.remote_python_version < 3 else 'python%s-dev' % env.remote_python_version, - 'python-pip' - ], update=update) + if fabtools.system.distrib_id() == 'Ubuntu' and float(fabtools.system.distrib_release()) >= 18.04: + if env.remote_python_version < 3: + fabric.api.sudo('add-apt-repository universe') + + fabtools.require.deb.packages( + [ + 'python2-dev' if env.remote_python_version < 3 else 'python%s-dev' % env.remote_python_version, + 'python3-pip' if not env.remote_python_version < 3 else 'python2', + 'build-essential', + ], + update=update, + ) + + if env.remote_python_version < 3: + # Warning dangerous !!!!! + fabric.api.sudo('curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py') + fabric.api.sudo('python2 get-pip.py') + else: + fabtools.require.deb.packages( + [ + 'python-dev' if env.remote_python_version < 3 else 'python%s-dev' % env.remote_python_version, + 'python-pip', + ], + update=update, + ) fabtools.require.python.install('pip', upgrade=True, use_sudo=True) @@ -28,37 +49,29 @@ def application_dependencies(upgrade_pkg, staging=True): with fabtools.python.virtualenv(env.remote_virtualenv_dir): with fabric.api.cd(env.remote_current_path): - requirements_file = os.path.join('requirements', - '%s.txt' % env.goal) if staging else 'requirements.txt' + requirements_file = os.path.join('requirements', '%s.txt' % env.goal) if staging else 'requirements.txt' # ugly fix for error when pip install fail and error raises while /home/user/.pip not writable pip_log = '%s/pip_error.log' % env.remote_home pip_cmd = 'pip --log-file %s' % pip_log if 'oracle_client_version' in env: - oracle_dir = 'instantclient_%s' % '_'.join( - env.oracle_client_version.split('.')[:2]) - oracle_root_path = os.path.join( - env.oracle_remote_dir, oracle_dir) - oracle_full_path = os.path.join( - env.remote_home, oracle_root_path) + oracle_dir = 'instantclient_%s' % '_'.join(env.oracle_client_version.split('.')[:2]) + oracle_root_path = os.path.join(env.oracle_remote_dir, oracle_dir) + oracle_full_path = os.path.join(env.remote_home, oracle_root_path) pip_cmd = 'ORACLE_HOME=%s pip' % oracle_full_path with shell("HOME=~%s %s" % (env.remote_owner, env.shell)): # upgrade pip to latest version - fabtools.require.python.install('pip', - upgrade=True, - use_sudo=True, - user=env.remote_owner, - pip_cmd=pip_cmd, - quiet=True) + fabtools.require.python.install( + 'pip', upgrade=True, use_sudo=True, user=env.remote_owner, pip_cmd=pip_cmd, quiet=True + ) - fabtools.python.install_requirements(requirements_file, - use_sudo=True, - user=env.remote_owner, - upgrade=upgrade_pkg, - pip_cmd='%s --no-cache-dir' % pip_cmd, - quiet=True) - - fabric.api.sudo( - 'pip install --log-file %s --quiet -e .' % pip_log , + fabtools.python.install_requirements( + requirements_file, + use_sudo=True, user=env.remote_owner, - pty=False) + upgrade=upgrade_pkg, + pip_cmd='%s --no-cache-dir' % pip_cmd, + quiet=True, + ) + + fabric.api.sudo('pip install --log-file %s --quiet -e .' % pip_log, user=env.remote_owner, pty=False) diff --git a/pydiploy/require/system.py b/pydiploy/require/system.py index add2d99..7bc9292 100644 --- a/pydiploy/require/system.py +++ b/pydiploy/require/system.py @@ -6,6 +6,7 @@ """ from contextlib import contextmanager + import fabric import fabtools from fabric.api import env @@ -27,16 +28,13 @@ def add_user(commands=None): """ fabtools.require.group(env.remote_group) - fabtools.require.user(env.remote_owner, - create_home=True, - create_group=True, - group=env.remote_group, - shell='/bin/bash') + fabtools.require.user( + env.remote_owner, create_home=True, create_group=True, group=env.remote_group, shell='/bin/bash' + ) if commands: - fabtools.require.sudoer('%%%s' % env.remote_group, - operators='%s,root' % env.remote_owner, - passwd=False, - commands=commands) + fabtools.require.sudoer( + '%%%s' % env.remote_group, operators='%s,root' % env.remote_owner, passwd=False, commands=commands + ) @do_verbose @@ -59,7 +57,7 @@ def set_locale(): Sets server's locales """ locale = fabric.api.run("echo $LANG") - if(locale != env.locale): + if locale != env.locale: fabric.api.sudo('locale-gen ' + env.locale) fabric.api.sudo('/usr/sbin/update-locale LANG=' + env.locale) @@ -69,7 +67,7 @@ def set_timezone(): """ Sets the timezone """ - if fabtools.system.distrib_id() not in('Ubuntu', 'Debian'): + if fabtools.system.distrib_id() not in ('Ubuntu', 'Debian'): print("Cannot deploy to non-debian/ubuntu host") return @@ -85,12 +83,11 @@ def permissions(): Makes the release group-writable """ - fabric.api.sudo("chown -R %(user)s:%(group)s %(domain_path)s" % - {'domain_path': env.remote_project_dir, - 'user': env.remote_owner, - 'group': env.remote_group}) - fabric.api.sudo("chmod -R g+w %(domain_path)s" % - {'domain_path': env.remote_project_dir}) + fabric.api.sudo( + "chown -R %(user)s:%(group)s %(domain_path)s" + % {'domain_path': env.remote_project_dir, 'user': env.remote_owner, 'group': env.remote_group} + ) + fabric.api.sudo("chmod -R g+w %(domain_path)s" % {'domain_path': env.remote_project_dir}) @do_verbose @@ -114,14 +111,14 @@ def check_python3_install(version='python3', update=False): Installs python 3 on ubuntu remote server """ - if not package_installed(version): - # TODO check for others ubuntu"s versions !!!!! - if fabtools.system.distrib_id() == 'Ubuntu' and float(fabtools.system.distrib_release()) < 13.10 or float(fabtools.system.distrib_release()) >= 16.04: - fabtools.require.deb.packages(['software-properties-common'], - update=update) - # Install mighty PPA - fabtools.require.deb.ppa('ppa:deadsnakes/ppa') - fabtools.require.deb.package(version, update=True) + # if not package_installed(version): + # # TODO check for others ubuntu"s versions !!!!! + # if fabtools.system.distrib_id() == 'Ubuntu' and float(fabtools.system.distrib_release()) < 13.10 or float(fabtools.system.distrib_release()) >= 16.04: + # #fabtools.require.deb.packages(['software-properties-common'], + # # update=update) + # # Install mighty PPA + # #fabtools.require.deb.ppa('ppa:deadsnakes/ppa') + fabtools.require.deb.package(version, update=True) @do_verbose From eb00e8630bd3ceb767194277f1abe57df529183d Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Wed, 3 Jun 2020 23:34:47 +0200 Subject: [PATCH 37/44] Enhance ubuntu 20.04 support --- pydiploy/require/circus.py | 100 ++++++++++++++++--------------- pydiploy/require/python/utils.py | 1 + pydiploy/require/system.py | 19 +++--- 3 files changed, 63 insertions(+), 57 deletions(-) diff --git a/pydiploy/require/circus.py b/pydiploy/require/circus.py index 48e5df4..f09e680 100644 --- a/pydiploy/require/circus.py +++ b/pydiploy/require/circus.py @@ -17,8 +17,9 @@ import fabric import fabtools -from fabric.api import env, warn_only, hide +from fabric.api import env, hide, warn_only from pydiploy.decorators import do_verbose + from .system import is_systemd @@ -27,24 +28,19 @@ def circus_pkg(update=False): """ Installs packages relatives to circus """ # TOOLD ! - # install ubuntu ppa for libzmq-dev if ubuntu <= 10.04 - #if fabtools.system.distrib_id() == 'Ubuntu' and fabtools.system.distrib_release() == '10.04': + # install ubuntu ppa for libzmq-dev if ubuntu <= 10.04 + # if fabtools.system.distrib_id() == 'Ubuntu' and fabtools.system.distrib_release() == '10.04': # fabtools.require.deb.packages(['python-software-properties'], # fabtools.require.deb.ppa('ppa:chris-lea/zeromq') # fabtools.require.deb.ppa('ppa:chris-lea/libpgm') if fabtools.system.distrib_id() == 'Ubuntu' and fabtools.system.distrib_release() >= '18.04': - fabtools.require.deb.packages([ - 'libzmq3-dev', - 'libevent-dev'], update=update) + fabtools.require.deb.packages(['libzmq3-dev', 'libevent-dev'], update=update) if fabtools.system.distrib_id() == 'Ubuntu' and fabtools.system.distrib_release() < '18.04': - fabtools.require.deb.packages([ - 'libzmq-dev', - 'libevent-dev'], update=update) + fabtools.require.deb.packages(['libzmq-dev', 'libevent-dev'], update=update) - fabtools.require.python.install(env.get('circus_package_name', 'circus'), - use_sudo=True, upgrade=update) + fabtools.require.python.install(env.get('circus_package_name', 'circus'), use_sudo=True, upgrade=update) if 'no_circus_web' not in env or not env.no_circus_web: fabtools.require.python.install('circus-web', use_sudo=True, upgrade=update) @@ -57,15 +53,15 @@ def circus_pkg(update=False): # base configuration file for circus fabtools.files.upload_template( 'circus.ini.tpl', - os.path.join(env.remote_home, - '.circus.ini'), + os.path.join(env.remote_home, '.circus.ini'), context=env, template_dir=os.path.join(env.lib_path, 'templates'), use_sudo=True, user=env.remote_owner, chown=True, mode='644', - use_jinja=True) + use_jinja=True, + ) # root directory for circus applications configuration fabtools.require.files.directory( @@ -73,7 +69,8 @@ def circus_pkg(update=False): use_sudo=True, owner=env.remote_owner, group=env.remote_group, - mode='750') + mode='750', + ) @do_verbose @@ -82,17 +79,17 @@ def app_circus_conf(): Sets circus app's configuration using templates in templates dir """ - fabtools.files.upload_template('app.ini.tpl', - os.path.join(env.remote_home, '.circus.d', - '%s.ini' % env.application_name), - context=env, - template_dir=os.path.join( - env.lib_path, 'templates'), - use_sudo=True, - user=env.remote_owner, - chown=True, - mode='644', - use_jinja=True) + fabtools.files.upload_template( + 'app.ini.tpl', + os.path.join(env.remote_home, '.circus.d', '%s.ini' % env.application_name), + context=env, + template_dir=os.path.join(env.lib_path, 'templates'), + use_sudo=True, + user=env.remote_owner, + chown=True, + mode='644', + use_jinja=True, + ) @do_verbose @@ -102,31 +99,33 @@ def upstart(): """ # Systemd if is_systemd(): - # init files to declare circus as a systemd daemon - fabtools.files.upload_template('circus.service.tpl', - '/etc/systemd/system/circus.service', - context=env, - template_dir=os.path.join( - env.lib_path, 'templates'), - use_sudo=True, - user='root', - chown=True, - mode='644', - use_jinja=True) + #  init files to declare circus as a systemd daemon + fabtools.files.upload_template( + 'circus.service.tpl', + '/etc/systemd/system/circus.service', + context=env, + template_dir=os.path.join(env.lib_path, 'templates'), + use_sudo=True, + user='root', + chown=True, + mode='644', + use_jinja=True, + ) fabric.api.sudo('systemctl daemon-reload') # Upstart else: - # init files to declare circus as an upstart daemon - fabtools.files.upload_template('upstart.conf.tpl', - '/etc/init/circus.conf', - context=env, - template_dir=os.path.join( - env.lib_path, 'templates'), - use_sudo=True, - user='root', - chown=True, - mode='644', - use_jinja=True) + #  init files to declare circus as an upstart daemon + fabtools.files.upload_template( + 'upstart.conf.tpl', + '/etc/init/circus.conf', + context=env, + template_dir=os.path.join(env.lib_path, 'templates'), + use_sudo=True, + user='root', + chown=True, + mode='644', + use_jinja=True, + ) @do_verbose @@ -150,5 +149,8 @@ def app_reload(): else: with fabric.api.settings(sudo_user=env.remote_owner): fabric.api.sudo('circusctl reloadconfig') - with warn_only(), hide(): + app_installed = fabtools.files.is_file( + path=os.path.join(env.remote_home, '.circus.d', '%s.ini' % env.application_name), use_sudo=True + ) + with warn_only(), hide('everything'): fabric.api.sudo('circusctl restart %s' % env.application_name) diff --git a/pydiploy/require/python/utils.py b/pydiploy/require/python/utils.py index 65cdd10..0701866 100644 --- a/pydiploy/require/python/utils.py +++ b/pydiploy/require/python/utils.py @@ -51,6 +51,7 @@ def application_dependencies(upgrade_pkg, staging=True): with fabric.api.cd(env.remote_current_path): requirements_file = os.path.join('requirements', '%s.txt' % env.goal) if staging else 'requirements.txt' # ugly fix for error when pip install fail and error raises while /home/user/.pip not writable + # TODO; maybe use app_name_pip_error.log file ? Or use a setting path for log file ? pip_log = '%s/pip_error.log' % env.remote_home pip_cmd = 'pip --log-file %s' % pip_log if 'oracle_client_version' in env: diff --git a/pydiploy/require/system.py b/pydiploy/require/system.py index 7bc9292..d362d37 100644 --- a/pydiploy/require/system.py +++ b/pydiploy/require/system.py @@ -111,14 +111,17 @@ def check_python3_install(version='python3', update=False): Installs python 3 on ubuntu remote server """ - # if not package_installed(version): - # # TODO check for others ubuntu"s versions !!!!! - # if fabtools.system.distrib_id() == 'Ubuntu' and float(fabtools.system.distrib_release()) < 13.10 or float(fabtools.system.distrib_release()) >= 16.04: - # #fabtools.require.deb.packages(['software-properties-common'], - # # update=update) - # # Install mighty PPA - # #fabtools.require.deb.ppa('ppa:deadsnakes/ppa') - fabtools.require.deb.package(version, update=True) + if not package_installed(version): + # TODO check for others ubuntu"s versions !!!!! + if float(fabtools.system.distrib_release()) >= 14.04: + # add-apt-repository moved to software-properties-common in 14.04 + fabtools.require.deb.packages('software-properties-common') + else: + fabtools.require.deb.packages('python-software-properties') + + # Install mighty PPA + fabtools.require.deb.ppa('ppa:deadsnakes/ppa') + fabtools.require.deb.package(version, update=True) @do_verbose From b3319a81cfd96a0cf96f5508df4d5f92a40d2a0d Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Thu, 4 Jun 2020 10:18:46 +0200 Subject: [PATCH 38/44] Enhance ubuntu 20.04 support & add no_tag_check param --- pydiploy/examples/bottle_fabfile.py | 45 ++-- pydiploy/examples/django_fabfile.py | 59 ++--- pydiploy/examples/simple_fabfile.py | 33 ++- pydiploy/params.py | 370 ++++++++++++++-------------- pydiploy/prepare.py | 2 - pydiploy/require/git.py | 8 +- pydiploy/require/system.py | 14 +- 7 files changed, 271 insertions(+), 260 deletions(-) diff --git a/pydiploy/examples/bottle_fabfile.py b/pydiploy/examples/bottle_fabfile.py index ca1184f..40e7863 100644 --- a/pydiploy/examples/bottle_fabfile.py +++ b/pydiploy/examples/bottle_fabfile.py @@ -4,28 +4,28 @@ Fabfile example file to deploy a standard bottle webapp """ -from fabric.api import (env, roles, execute, task) from os.path import join -from pydiploy.prepare import (tag as pydiploy_tag, - build_env as pydiploy_build_env) -from pydiploy.bottle import (deploy_backend as pydiploy_deploy_backend, - deploy_frontend as pydiploy_deploy_frontend, - rollback as pydiploy_rollback, - post_install_backend as pydiploy_postinstall_backend, - post_install_frontend as pydiploy_postinstall_frontend, - pre_install_backend as pydiploy_preinstall_backend, - pre_install_frontend as pydiploy_preinstall_frontend, - reload_frontend as pydiploy_reload_frontend, - reload_backend as pydiploy_reload_backend, - set_app_up as pydiploy_set_up, - set_app_down as pydiploy_set_down, - install_oracle_client as pydiploy_setup_oracle, - install_postgres_server as pydiploy_setup_postgres) +from fabric.api import env, execute, roles, task +from pydiploy.bottle import deploy_backend as pydiploy_deploy_backend +from pydiploy.bottle import deploy_frontend as pydiploy_deploy_frontend +from pydiploy.bottle import install_oracle_client as pydiploy_setup_oracle +from pydiploy.bottle import install_postgres_server as pydiploy_setup_postgres +from pydiploy.bottle import post_install_backend as pydiploy_postinstall_backend +from pydiploy.bottle import post_install_frontend as pydiploy_postinstall_frontend +from pydiploy.bottle import pre_install_backend as pydiploy_preinstall_backend +from pydiploy.bottle import pre_install_frontend as pydiploy_preinstall_frontend +from pydiploy.bottle import reload_backend as pydiploy_reload_backend +from pydiploy.bottle import reload_frontend as pydiploy_reload_frontend +from pydiploy.bottle import rollback as pydiploy_rollback +from pydiploy.bottle import set_app_down as pydiploy_set_down +from pydiploy.bottle import set_app_up as pydiploy_set_up +from pydiploy.prepare import build_env as pydiploy_build_env +from pydiploy.prepare import tag as pydiploy_tag # edit config here ! -env.use_sudo = True # use sudo or not +env.use_sudo = True # use sudo or not env.remote_owner = 'myremoteuser' env.remote_group = 'myremotegroup' @@ -36,8 +36,7 @@ env.remote_home = '/home/myremoteuser' # remote home root env.remote_python_version = 3.4 # python version env.remote_virtualenv_root = join(env.remote_home, '.virtualenvs') # venv root -env.remote_virtualenv_dir = join(env.remote_virtualenv_root, - env.application_name) # venv for webapp dir +env.remote_virtualenv_dir = join(env.remote_virtualenv_root, env.application_name) # venv for webapp dir env.remote_repo_url = 'git@git.net:myapp.git' env.remote_static_root = '/var/www/static' # root of static files @@ -85,7 +84,7 @@ # env.socket_host='localhost' # use it in env method to force a socket host -# env.run_tests_command = 'tox' +#  env.run_tests_command = 'tox' # env.media_folder = '/media' # path of the application's media files # env.remote_media_folder = '/srv/media/myapp' # remote folder of the application's media files @@ -96,6 +95,8 @@ # env.default_db_user = 'myapp_db_user' # env.default_db_password = 'S3CR3T' +# env.no_tag_check = True # When your fabfile is not in your project git repository be sure to set this var + @task def test(): @@ -150,6 +151,7 @@ def prod(): } execute(build_env) + # dont touch after that point if you don't know what you are doing ! @@ -200,7 +202,7 @@ def deploy(): execute(deploy_backend) execute(deploy_frontend) # uncomment this to toggle app to up mode again : - #execute(set_up) + # execute(set_up) @roles('web') @@ -244,6 +246,7 @@ def post_install_frontend(): """Post installation of frontend""" execute(pydiploy_postinstall_frontend) + @roles('web') @task def install_oracle(): diff --git a/pydiploy/examples/django_fabfile.py b/pydiploy/examples/django_fabfile.py index 3cb3dc0..67f8fd4 100644 --- a/pydiploy/examples/django_fabfile.py +++ b/pydiploy/examples/django_fabfile.py @@ -4,42 +4,40 @@ Fabfile example file to deploy a standard django webapp """ -from fabric.api import (env, roles, execute, task) from os.path import join -from pydiploy.prepare import (tag as pydiploy_tag, - build_env as pydiploy_build_env) -from pydiploy.django import (deploy_backend as pydiploy_deploy_backend, - deploy_frontend as pydiploy_deploy_frontend, - rollback as pydiploy_rollback, - post_install_backend as pydiploy_postinstall_backend, - post_install_frontend as pydiploy_postinstall_frontend, - pre_install_backend as pydiploy_preinstall_backend, - pre_install_frontend as pydiploy_preinstall_frontend, - reload_frontend as pydiploy_reload_frontend, - reload_backend as pydiploy_reload_backend, - set_app_up as pydiploy_set_up, - set_app_down as pydiploy_set_down, - custom_manage_command as pydiploy_custom_command, - install_oracle_client as pydiploy_setup_oracle, - install_postgres_server as pydiploy_setup_postgres) - +from fabric.api import env, execute, roles, task +from pydiploy.django import custom_manage_command as pydiploy_custom_command +from pydiploy.django import deploy_backend as pydiploy_deploy_backend +from pydiploy.django import deploy_frontend as pydiploy_deploy_frontend +from pydiploy.django import install_oracle_client as pydiploy_setup_oracle +from pydiploy.django import install_postgres_server as pydiploy_setup_postgres +from pydiploy.django import post_install_backend as pydiploy_postinstall_backend +from pydiploy.django import post_install_frontend as pydiploy_postinstall_frontend +from pydiploy.django import pre_install_backend as pydiploy_preinstall_backend +from pydiploy.django import pre_install_frontend as pydiploy_preinstall_frontend +from pydiploy.django import reload_backend as pydiploy_reload_backend +from pydiploy.django import reload_frontend as pydiploy_reload_frontend +from pydiploy.django import rollback as pydiploy_rollback +from pydiploy.django import set_app_down as pydiploy_set_down +from pydiploy.django import set_app_up as pydiploy_set_up +from pydiploy.prepare import build_env as pydiploy_build_env +from pydiploy.prepare import tag as pydiploy_tag # edit config here ! -env.use_sudo = True # use sudo or not +env.use_sudo = True # use sudo or not env.remote_owner = 'django' # remote server user env.remote_group = 'django' # remote server group -env.application_name = 'myapp' # name of webapp +env.application_name = 'myapp' # name of webapp env.root_package_name = 'myapp' # name of app in webapp env.remote_home = '/home/django' # remote home root env.remote_python_version = 3.4 # python version env.remote_virtualenv_root = join(env.remote_home, '.virtualenvs') # venv root -env.remote_virtualenv_dir = join(env.remote_virtualenv_root, - env.application_name) # venv for webapp dir +env.remote_virtualenv_dir = join(env.remote_virtualenv_root, env.application_name) # venv for webapp dir env.remote_repo_url = 'git@git.net:myapp.git' # git repository url env.local_tmp_dir = '/tmp' # tmp dir env.remote_static_root = '/var/www/static' # root of static files @@ -86,7 +84,7 @@ # env.socket_host='localhost' # use it in env method to force a socket host -# env.run_tests_command = 'tox' +#  env.run_tests_command = 'tox' # env.media_folder = '/media' # path of the application's media files # env.remote_media_folder = '/srv/media/myapp' # remote folder of the application's media files @@ -97,6 +95,8 @@ # env.default_db_user = 'myapp_db_user' # env.default_db_password = 'S3CR3T' +# env.no_tag_check = True # When your fabfile is not in your project git repository be sure to set this var + @task def test(): @@ -111,7 +111,7 @@ def test(): env.short_server_name = 'myapp-dev' env.static_folder = '/site_media/' env.server_ip = '192.168.1.3' - #env.port = '1664' + # env.port = '1664' env.no_shared_sessions = False env.server_ssl_on = False env.goal = 'test' @@ -136,7 +136,7 @@ def prod(): env.short_server_name = 'myapp' env.static_folder = '/site_media/' env.server_ip = '' - #env.port = '' + # env.port = '' env.no_shared_sessions = False env.server_ssl_on = True env.path_to_cert = '/etc/ssl/certs/myapp.net.pem' @@ -153,8 +153,10 @@ def prod(): } execute(build_env) + # dont touch after that point if you don't know what you are doing ! + @task def tag(version_string): """ Set the version to deploy to `version_number`. """ @@ -167,7 +169,7 @@ def head_master(): execute(pydiploy_tag, version='master') -@roles(['web','lb']) +@roles(['web', 'lb']) def build_env(): execute(pydiploy_build_env) @@ -201,7 +203,7 @@ def deploy(): execute(deploy_backend) execute(deploy_frontend) # uncomment this to toggle app to up mode again : - #execute(set_up) + # execute(set_up) @roles('web') @@ -297,4 +299,5 @@ def set_up(): @task def custom_manage_cmd(cmd): """ Execute custom command in manage.py """ - execute(pydiploy_custom_command,cmd) + execute(pydiploy_custom_command, cmd) + diff --git a/pydiploy/examples/simple_fabfile.py b/pydiploy/examples/simple_fabfile.py index e4983c0..ea95d33 100644 --- a/pydiploy/examples/simple_fabfile.py +++ b/pydiploy/examples/simple_fabfile.py @@ -4,19 +4,19 @@ Fabfile example file to deploy a standard simple webapp """ -from fabric.api import (env, roles, execute, task) from os.path import join -from pydiploy.prepare import (tag as pydiploy_tag, - build_env as pydiploy_build_env) -from pydiploy.simple import (deploy_backend as pydiploy_deploy_backend, - rollback as pydiploy_rollback, - post_install_backend as pydiploy_postinstall_backend, - pre_install_backend as pydiploy_preinstall_backend) +from fabric.api import env, execute, roles, task +from pydiploy.prepare import build_env as pydiploy_build_env +from pydiploy.prepare import tag as pydiploy_tag +from pydiploy.simple import deploy_backend as pydiploy_deploy_backend +from pydiploy.simple import post_install_backend as pydiploy_postinstall_backend +from pydiploy.simple import pre_install_backend as pydiploy_preinstall_backend +from pydiploy.simple import rollback as pydiploy_rollback # edit config here ! -env.use_sudo = True # use sudo or not +env.use_sudo = True # use sudo or not env.remote_owner = 'myremoteuser' env.remote_group = 'myremotegroup' @@ -27,8 +27,7 @@ env.remote_home = '/home/myremoteuser' # remote home root env.remote_python_version = 3.4 # python version env.remote_virtualenv_root = join(env.remote_home, '.virtualenvs') # venv root -env.remote_virtualenv_dir = join(env.remote_virtualenv_root, - env.application_name) # venv for webapp dir +env.remote_virtualenv_dir = join(env.remote_virtualenv_root, env.application_name) # venv for webapp dir env.remote_repo_url = 'git@git.net:myapp.git' env.local_tmp_dir = '/tmp' # tmp dir @@ -53,7 +52,7 @@ # env.verbose = True # verbose display for pydiploy default value = True env.req_pydiploy_version = '1.1.5.0' # required pydiploy version for this fabfile -# env.run_tests_command = 'tox' +#  env.run_tests_command = 'tox' # fill and uncomment not to pass parameters in term (eg: fab tag:master test --set default_db_host='localhost',default_db_name='my_app_db' ) # env.default_db_host = 'localhost' @@ -61,14 +60,14 @@ # env.default_db_user = 'myapp_db_user' # env.default_db_password = 'S3CR3T' +# env.no_tag_check = True # When your fabfile is not in your project git repository be sure to set this var + @task def test(): """Define test stage""" env.user = 'vagrant' - env.roledefs = { - 'web': ['192.168.1.4'] - } + env.roledefs = {'web': ['192.168.1.4']} env.goal = "test" # env.server_name = "myapp.net" # optional: if you want to use an url for the name of the remote app folder instead of the application name (manual bottle or flask app) env.map_settings = { @@ -78,12 +77,11 @@ def test(): } execute(build_env) + @task def prod(): """Define prod stage""" - env.roledefs = { - 'web': ['myserver.net'] - } + env.roledefs = {'web': ['myserver.net']} env.goal = "prod" # env.server_name = "myapp.net" # optional: if you want to use an url for the name of the remote app folder instead of the application name (manual bottle or flask app) env.map_settings = { @@ -93,6 +91,7 @@ def prod(): } execute(build_env) + # dont touch after that point if you don't know what you are doing ! diff --git a/pydiploy/params.py b/pydiploy/params.py index 9935a99..cba5e40 100644 --- a/pydiploy/params.py +++ b/pydiploy/params.py @@ -19,192 +19,194 @@ PARAMS = { 'default': { - - 'required_params': {'user': "user for ssh", - 'remote_owner': "remote server user", - 'remote_group': "remote server user group", - 'application_name': "name of wepapp", - 'root_package_name': "name of app in webapp", - 'remote_home': "remote home root", - 'remote_python_version': "remote python version to use", - 'remote_virtualenv_root': "remote virtualenv root", - 'remote_virtualenv_dir': "remote virtualenv dir for wepapp", - 'remote_repo_url': "git repository url", - 'local_tmp_dir': "local tmp dir", - 'remote_static_root': "root of static files", - 'locale': "locale to use on remote", - 'timezone': "timezone used on remote", - 'keep_releases': "number of old releases to keep", - 'roledefs': "Role to use to deploy", - 'backends': "backend to use to deploy", - 'server_name': "name of webserver", - 'short_server_name': "short name of webserver", - 'static_folder': "path of static folder", - 'goal': "stage to use to deploy (dev,prod,test...)", - 'socket_port': "port to use for socket"}, - - 'optional_params': {'excluded_files': "file(s) to exclude when deploying", - 'extra_ppa_to_install': "extra ppa(s) to install on remote", - 'extra_source_to_install': "extra debian source(s) to install on remote", - 'extra_pkg_to_install': "extra package(s) to install on remote", - 'cfg_shared_files': "shared file(s) to deploy in shared dir", - 'extra_goals': "extra goal(s) to add to deploy", - 'oracle_client_version': "oracle client version to install", - 'oracle_download_url': "oracle client download url", - 'oracle_remote_dir': "oracle remote directory", - 'oracle_packages': "oracle packages to install", - 'circus_package_name': "circus package name", - 'dest_path': "destination path", - 'nginx_location_extra_directives': "nginx location extra directive(s)", - 'verbose_output': "verbose output (True/False)", - 'no_config_test': "don't check config params (True/False)", - 'req_pydiploy_version': "pydiploy version required for checking fabfile syntax", - 'extra_symlink_dirs': 'extra dirs to symlink to shared folder', - 'tag': 'tag to deploy', - 'no_shared_sessions': 'if True add ip_hash to nginx config', - 'server_ssl_on': 'use ssl on nginx', - 'path_to_cert': 'ssl certificate path', - 'path_to_cert_key': 'ssl cert key path', - 'map_settings': 'settings to map in django settings files', - 'maintenance_title': 'customize maintenance page title', - 'maintenance_text': 'customize maintenance page text', - 'nginx_force_start': 'if True, it forces to start nginx when nginx is not started', - 'socket_host': 'force a socket host', - 'no_circus_web': 'provides a way to not install circus-web', - 'circus_backend': 'name of the backend to use with circus', - 'chaussette_backend': 'name of the backend to use with chaussette', - 'run_tests_command': 'name of the command to run tests', - 'remote_repo_specific_folder': 'specify a subfolder for the remote repository', - 'remote_media_folder': "remote folder of the application's media files", - 'media_folder': "path of the application's media files"} - + 'required_params': { + 'user': "user for ssh", + 'remote_owner': "remote server user", + 'remote_group': "remote server user group", + 'application_name': "name of wepapp", + 'root_package_name': "name of app in webapp", + 'remote_home': "remote home root", + 'remote_python_version': "remote python version to use", + 'remote_virtualenv_root': "remote virtualenv root", + 'remote_virtualenv_dir': "remote virtualenv dir for wepapp", + 'remote_repo_url': "git repository url", + 'local_tmp_dir': "local tmp dir", + 'remote_static_root': "root of static files", + 'locale': "locale to use on remote", + 'timezone': "timezone used on remote", + 'keep_releases': "number of old releases to keep", + 'roledefs': "Role to use to deploy", + 'backends': "backend to use to deploy", + 'server_name': "name of webserver", + 'short_server_name': "short name of webserver", + 'static_folder': "path of static folder", + 'goal': "stage to use to deploy (dev,prod,test...)", + 'socket_port': "port to use for socket", + }, + 'optional_params': { + 'excluded_files': "file(s) to exclude when deploying", + 'extra_ppa_to_install': "extra ppa(s) to install on remote", + 'extra_source_to_install': "extra debian source(s) to install on remote", + 'extra_pkg_to_install': "extra package(s) to install on remote", + 'cfg_shared_files': "shared file(s) to deploy in shared dir", + 'extra_goals': "extra goal(s) to add to deploy", + 'oracle_client_version': "oracle client version to install", + 'oracle_download_url': "oracle client download url", + 'oracle_remote_dir': "oracle remote directory", + 'oracle_packages': "oracle packages to install", + 'circus_package_name': "circus package name", + 'dest_path': "destination path", + 'nginx_location_extra_directives': "nginx location extra directive(s)", + 'verbose_output': "verbose output (True/False)", + 'no_config_test': "don't check config params (True/False)", + 'req_pydiploy_version': "pydiploy version required for checking fabfile syntax", + 'extra_symlink_dirs': 'extra dirs to symlink to shared folder', + 'tag': 'tag to deploy', + 'no_shared_sessions': 'if True add ip_hash to nginx config', + 'server_ssl_on': 'use ssl on nginx', + 'path_to_cert': 'ssl certificate path', + 'path_to_cert_key': 'ssl cert key path', + 'map_settings': 'settings to map in django settings files', + 'maintenance_title': 'customize maintenance page title', + 'maintenance_text': 'customize maintenance page text', + 'nginx_force_start': 'if True, it forces to start nginx when nginx is not started', + 'socket_host': 'force a socket host', + 'no_circus_web': 'provides a way to not install circus-web', + 'circus_backend': 'name of the backend to use with circus', + 'chaussette_backend': 'name of the backend to use with chaussette', + 'run_tests_command': 'name of the command to run tests', + 'remote_repo_specific_folder': 'specify a subfolder for the remote repository', + 'remote_media_folder': "remote folder of the application's media files", + 'media_folder': "path of the application's media files", + 'no_tag_check': "If True no tag/branch check when deploying", + }, }, 'simple': { - - 'required_params': {'user': "user for ssh", - 'remote_owner': "remote server user", - 'remote_group': "remote server user group", - 'application_name': "name of wepapp", - 'root_package_name': "name of app in webapp", - 'remote_home': "remote home root", - 'remote_python_version': "remote python version to use", - 'remote_virtualenv_root': "remote virtualenv root", - 'remote_virtualenv_dir': "remote virtualenv dir for wepapp", - 'remote_repo_url': "git repository url", - 'local_tmp_dir': "local tmp dir", - # 'remote_static_root': "root of static files", - 'locale': "locale to use on remote", - 'timezone': "timezone used on remote", - 'keep_releases': "number of old releases to keep", - 'roledefs': "Role to use to deploy", - # 'backends': "backend to use to deploy", - # 'server_name': "name of webserver", - # 'short_server_name': "short name of webserver", - # 'static_folder': "path of static folder", - 'goal': "stage to use to deploy (dev,prod,test...)" - # 'socket_port': "port to use for socket" - }, - - 'optional_params': {'excluded_files': "file(s) to exclude when deploying", - 'extra_ppa_to_install': "extra ppa(s) to install on remote", - 'extra_source_to_install': "extra debian source(s) to install on remote", - 'extra_pkg_to_install': "extra package(s) to install on remote", - 'cfg_shared_files': "shared file(s) to deploy in shared dir", - 'extra_goals': "extra goal(s) to add to deploy", - # 'oracle_client_version': "oracle client version to install", - # 'oracle_download_url': "oracle client download url", - # 'oracle_remote_dir': "oracle remote directory", - # 'oracle_packages': "oracle packages to install", - # 'circus_package_name': "circus package name", - 'dest_path': "destination path", - # 'nginx_location_extra_directives': "nginx location extra directive(s)", - 'verbose_output': "verbose output (True/False)", - 'no_config_test': "don't check config params (True/False)", - 'req_pydiploy_version': "pydiploy version required for checking fabfile syntax", - 'extra_symlink_dirs': 'extra dirs to symlink to shared folder', - 'tag': 'tag to deploy', - # 'no_shared_sessions': 'if True add ip_hash to nginx config', - # 'server_ssl_on': 'use ssl on nginx', - # 'path_to_cert': 'ssl certificate path', - # 'path_to_cert_key': 'ssl cert key path', - 'map_settings': 'settings to map in django settings files', - # 'maintenance_title': 'customize maintenance page title', - # 'maintenance_text': 'customize maintenance page text', - # 'nginx_force_start': 'if True, it forces to start nginx when nginx is not started', - # 'socket_host': 'force a socket host', - # 'no_circus_web': 'provides a way to not install circus-web', - # 'circus_backend': 'name of the backend to use with circus', - # 'chaussette_backend': 'name of the backend to use with chaussette', - 'run_tests_command': 'name of the command to run tests', - 'remote_repo_specific_folder': 'specify a subfolder for the remote repository', - # 'remote_media_folder': "remote folder of the application's media files" - # 'media_folder': "path of the application's media files" - 'server_name': "if you want to use an url for the name of the remote app folder instead of the application name (manual bottle or flask app)", - } - + 'required_params': { + 'user': "user for ssh", + 'remote_owner': "remote server user", + 'remote_group': "remote server user group", + 'application_name': "name of wepapp", + 'root_package_name': "name of app in webapp", + 'remote_home': "remote home root", + 'remote_python_version': "remote python version to use", + 'remote_virtualenv_root': "remote virtualenv root", + 'remote_virtualenv_dir': "remote virtualenv dir for wepapp", + 'remote_repo_url': "git repository url", + 'local_tmp_dir': "local tmp dir", + # 'remote_static_root': "root of static files", + 'locale': "locale to use on remote", + 'timezone': "timezone used on remote", + 'keep_releases': "number of old releases to keep", + 'roledefs': "Role to use to deploy", + # 'backends': "backend to use to deploy", + # 'server_name': "name of webserver", + # 'short_server_name': "short name of webserver", + # 'static_folder': "path of static folder", + 'goal': "stage to use to deploy (dev,prod,test...)" + # 'socket_port': "port to use for socket" + }, + 'optional_params': { + 'excluded_files': "file(s) to exclude when deploying", + 'extra_ppa_to_install': "extra ppa(s) to install on remote", + 'extra_source_to_install': "extra debian source(s) to install on remote", + 'extra_pkg_to_install': "extra package(s) to install on remote", + 'cfg_shared_files': "shared file(s) to deploy in shared dir", + 'extra_goals': "extra goal(s) to add to deploy", + # 'oracle_client_version': "oracle client version to install", + # 'oracle_download_url': "oracle client download url", + # 'oracle_remote_dir': "oracle remote directory", + # 'oracle_packages': "oracle packages to install", + # 'circus_package_name': "circus package name", + 'dest_path': "destination path", + # 'nginx_location_extra_directives': "nginx location extra directive(s)", + 'verbose_output': "verbose output (True/False)", + 'no_config_test': "don't check config params (True/False)", + 'req_pydiploy_version': "pydiploy version required for checking fabfile syntax", + 'extra_symlink_dirs': 'extra dirs to symlink to shared folder', + 'tag': 'tag to deploy', + # 'no_shared_sessions': 'if True add ip_hash to nginx config', + # 'server_ssl_on': 'use ssl on nginx', + # 'path_to_cert': 'ssl certificate path', + # 'path_to_cert_key': 'ssl cert key path', + 'map_settings': 'settings to map in django settings files', + # 'maintenance_title': 'customize maintenance page title', + # 'maintenance_text': 'customize maintenance page text', + # 'nginx_force_start': 'if True, it forces to start nginx when nginx is not started', + # 'socket_host': 'force a socket host', + # 'no_circus_web': 'provides a way to not install circus-web', + # 'circus_backend': 'name of the backend to use with circus', + # 'chaussette_backend': 'name of the backend to use with chaussette', + 'run_tests_command': 'name of the command to run tests', + 'remote_repo_specific_folder': 'specify a subfolder for the remote repository', + # 'remote_media_folder': "remote folder of the application's media files" + # 'media_folder': "path of the application's media files" + 'server_name': "if you want to use an url for the name of the remote app folder instead of the application name (manual bottle or flask app)", + 'no_tag_check': "If True no tag/branch check when deploying", + }, }, 'bottle': { - - 'required_params': {'user': "user for ssh", - 'remote_owner': "remote server user", - 'remote_group': "remote server user group", - 'application_name': "name of wepapp", - 'root_package_name': "name of app in webapp", - 'remote_home': "remote home root", - 'remote_python_version': "remote python version to use", - 'remote_virtualenv_root': "remote virtualenv root", - 'remote_virtualenv_dir': "remote virtualenv dir for wepapp", - 'remote_repo_url': "git repository url", - 'local_tmp_dir': "local tmp dir", - 'remote_static_root': "root of static files", - 'locale': "locale to use on remote", - 'timezone': "timezone used on remote", - 'keep_releases': "number of old releases to keep", - 'roledefs': "Role to use to deploy", - 'backends': "backend to use to deploy", - 'server_name': "name of webserver", - 'short_server_name': "short name of webserver", - 'static_folder': "path of static folder", - 'goal': "stage to use to deploy (dev,prod,test...)", - 'socket_port': "port to use for socket" - }, - - 'optional_params': {'excluded_files': "file(s) to exclude when deploying", - 'extra_ppa_to_install': "extra ppa(s) to install on remote", - 'extra_source_to_install': "extra debian source(s) to install on remote", - 'extra_pkg_to_install': "extra package(s) to install on remote", - 'cfg_shared_files': "shared file(s) to deploy in shared dir", - 'extra_goals': "extra goal(s) to add to deploy", - 'oracle_client_version': "oracle client version to install", - 'oracle_download_url': "oracle client download url", - 'oracle_remote_dir': "oracle remote directory", - 'oracle_packages': "oracle packages to install", - 'circus_package_name': "circus package name", - 'dest_path': "destination path", - 'nginx_location_extra_directives': "nginx location extra directive(s)", - 'verbose_output': "verbose output (True/False)", - 'no_config_test': "don't check config params (True/False)", - 'req_pydiploy_version': "pydiploy version required for checking fabfile syntax", - 'extra_symlink_dirs': 'extra dirs to symlink to shared folder', - 'tag': 'tag to deploy', - 'no_shared_sessions': 'if True add ip_hash to nginx config', - 'server_ssl_on': 'use ssl on nginx', - 'path_to_cert': 'ssl certificate path', - 'path_to_cert_key': 'ssl cert key path', - 'map_settings': 'settings to map in django settings files', - 'maintenance_title': 'customize maintenance page title', - 'maintenance_text': 'customize maintenance page text', - 'nginx_force_start': 'if True, it forces to start nginx when nginx is not started', - 'socket_host': 'force a socket host', - 'no_circus_web': 'provides a way to not install circus-web', - 'circus_backend': 'name of the backend to use with circus', - 'chaussette_backend': 'name of the backend to use with chaussette', - 'run_tests_command': 'name of the command to run tests', - 'remote_repo_specific_folder': 'specify a subfolder for the remote repository', - 'remote_media_folder': "remote folder of the application's media files", - 'media_folder': "path of the application's media files", - 'server_name': "if you want to use an url for the name of the remote app folder instead of the application name (manual bottle or flask app)", - } - - } + 'required_params': { + 'user': "user for ssh", + 'remote_owner': "remote server user", + 'remote_group': "remote server user group", + 'application_name': "name of wepapp", + 'root_package_name': "name of app in webapp", + 'remote_home': "remote home root", + 'remote_python_version': "remote python version to use", + 'remote_virtualenv_root': "remote virtualenv root", + 'remote_virtualenv_dir': "remote virtualenv dir for wepapp", + 'remote_repo_url': "git repository url", + 'local_tmp_dir': "local tmp dir", + 'remote_static_root': "root of static files", + 'locale': "locale to use on remote", + 'timezone': "timezone used on remote", + 'keep_releases': "number of old releases to keep", + 'roledefs': "Role to use to deploy", + 'backends': "backend to use to deploy", + 'server_name': "name of webserver", + 'short_server_name': "short name of webserver", + 'static_folder': "path of static folder", + 'goal': "stage to use to deploy (dev,prod,test...)", + 'socket_port': "port to use for socket", + }, + 'optional_params': { + 'excluded_files': "file(s) to exclude when deploying", + 'extra_ppa_to_install': "extra ppa(s) to install on remote", + 'extra_source_to_install': "extra debian source(s) to install on remote", + 'extra_pkg_to_install': "extra package(s) to install on remote", + 'cfg_shared_files': "shared file(s) to deploy in shared dir", + 'extra_goals': "extra goal(s) to add to deploy", + 'oracle_client_version': "oracle client version to install", + 'oracle_download_url': "oracle client download url", + 'oracle_remote_dir': "oracle remote directory", + 'oracle_packages': "oracle packages to install", + 'circus_package_name': "circus package name", + 'dest_path': "destination path", + 'nginx_location_extra_directives': "nginx location extra directive(s)", + 'verbose_output': "verbose output (True/False)", + 'no_config_test': "don't check config params (True/False)", + 'req_pydiploy_version': "pydiploy version required for checking fabfile syntax", + 'extra_symlink_dirs': 'extra dirs to symlink to shared folder', + 'tag': 'tag to deploy', + 'no_shared_sessions': 'if True add ip_hash to nginx config', + 'server_ssl_on': 'use ssl on nginx', + 'path_to_cert': 'ssl certificate path', + 'path_to_cert_key': 'ssl cert key path', + 'map_settings': 'settings to map in django settings files', + 'maintenance_title': 'customize maintenance page title', + 'maintenance_text': 'customize maintenance page text', + 'nginx_force_start': 'if True, it forces to start nginx when nginx is not started', + 'socket_host': 'force a socket host', + 'no_circus_web': 'provides a way to not install circus-web', + 'circus_backend': 'name of the backend to use with circus', + 'chaussette_backend': 'name of the backend to use with chaussette', + 'run_tests_command': 'name of the command to run tests', + 'remote_repo_specific_folder': 'specify a subfolder for the remote repository', + 'remote_media_folder': "remote folder of the application's media files", + 'media_folder': "path of the application's media files", + 'server_name': "if you want to use an url for the name of the remote app folder instead of the application name (manual bottle or flask app)", + 'no_tag_check': "If True no tag/branch check when deploying", + }, + }, } diff --git a/pydiploy/prepare.py b/pydiploy/prepare.py index 7389192..7af7645 100644 --- a/pydiploy/prepare.py +++ b/pydiploy/prepare.py @@ -18,7 +18,6 @@ import fabtools from fabric.api import env from pkg_resources import Requirement, resource_filename - from pydiploy.params import PARAMS from pydiploy.require.git import check_tag_exist from pydiploy.version import __version__, __version_info__ @@ -264,4 +263,3 @@ def process_releases(): 'releases_path': env.remote_releases_path, 'previous_revision': env.previous_revision, } - diff --git a/pydiploy/require/git.py b/pydiploy/require/git.py index 89d1e2f..631d0e7 100644 --- a/pydiploy/require/git.py +++ b/pydiploy/require/git.py @@ -10,12 +10,14 @@ import os import fabric +from fabric.api import env from pydiploy.decorators import do_verbose @do_verbose -def archive(filename, path='/tmp', format="tar.gz", tag="HEAD", remote="", - prefix="", project_path=".", specific_folder=""): +def archive( + filename, path='/tmp', format="tar.gz", tag="HEAD", remote="", prefix="", project_path=".", specific_folder="" +): """ Creates an archive from a git repository or directly from a project * Using at the root path of a local cloned repository. This will create @@ -118,6 +120,6 @@ def collect_branches(project_path='.', remote=""): def check_tag_exist(tag=None): """ Checks if a tag/branch exists in the repository """ if tag: - if (tag not in collect_branches() and tag not in collect_tags()): + if tag not in collect_branches() and tag not in collect_tags() and 'no_tag_check' not in env: return False return True diff --git a/pydiploy/require/system.py b/pydiploy/require/system.py index d362d37..50a4c19 100644 --- a/pydiploy/require/system.py +++ b/pydiploy/require/system.py @@ -113,14 +113,18 @@ def check_python3_install(version='python3', update=False): if not package_installed(version): # TODO check for others ubuntu"s versions !!!!! - if float(fabtools.system.distrib_release()) >= 14.04: + fabric.api.puts(fabtools.system.distrib_release()) + fabric.api.puts(fabtools.system.distrib_release() >= 14.04) + # Install mighty PPA + if fabtools.system.distrib_release() >= 14.04: # add-apt-repository moved to software-properties-common in 14.04 - fabtools.require.deb.packages('software-properties-common') + fabtools.require.deb.package('software-properties-common') + # deb.pap problem for 20.04 in fabtools + fabric.api.sudo('add-apt-repository --yes ppa:deadsnakes/ppa') else: - fabtools.require.deb.packages('python-software-properties') + fabtools.require.deb.package('python-software-properties') + fabtools.require.deb.ppa('ppa:deadsnakes/ppa') - # Install mighty PPA - fabtools.require.deb.ppa('ppa:deadsnakes/ppa') fabtools.require.deb.package(version, update=True) From 0cb138285bbed36a4b362657a7600de02b7f002b Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Thu, 4 Jun 2020 11:03:29 +0200 Subject: [PATCH 39/44] Remove problematic log date format not working --- pydiploy/require/circus.py | 2 +- pydiploy/templates/app.ini.tpl | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pydiploy/require/circus.py b/pydiploy/require/circus.py index f09e680..b8f7b97 100644 --- a/pydiploy/require/circus.py +++ b/pydiploy/require/circus.py @@ -152,5 +152,5 @@ def app_reload(): app_installed = fabtools.files.is_file( path=os.path.join(env.remote_home, '.circus.d', '%s.ini' % env.application_name), use_sudo=True ) - with warn_only(), hide('everything'): + with warn_only(), hide(): fabric.api.sudo('circusctl restart %s' % env.application_name) diff --git a/pydiploy/templates/app.ini.tpl b/pydiploy/templates/app.ini.tpl index f2c8c26..5344f4e 100644 --- a/pydiploy/templates/app.ini.tpl +++ b/pydiploy/templates/app.ini.tpl @@ -11,13 +11,16 @@ gid = {{ remote_group }} stderr_stream.class = FileStream stderr_stream.filename = {{ remote_shared_path }}/log/circus_error.log -stderr_stream.time_format = %Y-%m-%d %H:%M:%S +# FIXME ! +# stderr_stream.time_format = %Y-%m-%d %H:%M:%S stderr_stream.max_bytes = 209715200 stderr_stream.backup_count = 5 stdout_stream.class = FileStream stdout_stream.filename = {{ remote_shared_path }}/log/circus.log -stdout_stream.time_format = %Y-%m-%d %H:%M:%S + +# FIXME ! +# stdout_stream.time_format = %Y-%m-%d %H:%M:%S stdout_stream.max_bytes = 209715200 stdout_stream.backup_count = 5 @@ -31,3 +34,6 @@ host = {{ host }} port = {{ socket_port }} + +[env:{{ application_name }}] +#LANG=en_US.UTF-8 From aa35cef73ce7e900e19068f49bcb433dfa8a016e Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Thu, 4 Jun 2020 17:39:28 +0200 Subject: [PATCH 40/44] Fix tests --- pydiploy/require/releases_manager.py | 187 ++++++++++++++++----------- tests/test_require_circus.py | 168 ++++++++++++------------ tests/test_require_system.py | 79 ++++++----- 3 files changed, 247 insertions(+), 187 deletions(-) diff --git a/pydiploy/require/releases_manager.py b/pydiploy/require/releases_manager.py index 0f47f25..9cb7b6b 100644 --- a/pydiploy/require/releases_manager.py +++ b/pydiploy/require/releases_manager.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- +import json import os from time import time import fabric import fabtools import pydiploy - from fabric.api import env, warn_only from pydiploy.decorators import do_verbose @@ -16,9 +16,10 @@ def set_current(): """ Uses current directory for new release """ - fabric.api.sudo("ln -nfs %(current_release)s %(current_path)s" - % {'current_release': env.remote_current_release, - 'current_path': env.remote_current_path}) + fabric.api.sudo( + "ln -nfs %(current_release)s %(current_path)s" + % {'current_release': env.remote_current_release, 'current_path': env.remote_current_path} + ) @do_verbose @@ -26,16 +27,17 @@ def setup(): """ Configs stuff for deployement """ - fabric.api.sudo("mkdir -p %(remote_domain_path)s/{releases,shared}" % - {'remote_domain_path': env.remote_project_dir}) - fabric.api.sudo("mkdir -p %(remote_shared_path)s/{config,log}" % - {'remote_shared_path': env.remote_shared_path}) + fabric.api.sudo( + "mkdir -p %(remote_domain_path)s/{releases,shared}" % {'remote_domain_path': env.remote_project_dir} + ) + fabric.api.sudo("mkdir -p %(remote_shared_path)s/{config,log}" % {'remote_shared_path': env.remote_shared_path}) # extra symlinks if present in settings if env.has_key('extra_symlink_dirs'): for extra_symlink_dir in env.extra_symlink_dirs: - fabric.api.sudo("mkdir -p %(remote_shared_path)s/%(shared_dir)s" % - {'remote_shared_path': env.remote_shared_path, - 'shared_dir': os.path.basename(extra_symlink_dir)}) + fabric.api.sudo( + "mkdir -p %(remote_shared_path)s/%(shared_dir)s" + % {'remote_shared_path': env.remote_shared_path, 'shared_dir': os.path.basename(extra_symlink_dir)} + ) fabric.api.execute(pydiploy.require.system.permissions) @@ -49,13 +51,15 @@ def cleanup(): if 'releases' in env and len(env.releases) >= env.keep_releases: directories = env.releases directories.reverse() - del directories[:env.keep_releases] - env.directories = ' '.join(["%(releases_path)s/%(release)s" % - {'releases_path': env.remote_releases_path, - 'release': release} for release in directories]) + del directories[: env.keep_releases] + env.directories = ' '.join( + [ + "%(releases_path)s/%(release)s" % {'releases_path': env.remote_releases_path, 'release': release} + for release in directories + ] + ) - fabric.api.sudo("rm -rf %(directories)s" % - {'directories': env.directories}) + fabric.api.sudo("rm -rf %(directories)s" % {'directories': env.directories}) @do_verbose @@ -67,63 +71,75 @@ def deploy_code(): # checks if tag is specified if not fabric.api.prompt user if "tag" not in env: tag_requested = fabric.api.prompt('Please specify target tag used: ') - while(not pydiploy.require.git.check_tag_exist(tag_requested)): + while not pydiploy.require.git.check_tag_exist(tag_requested): tag_requested = fabric.api.prompt( - 'tag %s unknown please specify valid target tag used: ' % fabric.colors.red(tag_requested)) + 'tag %s unknown please specify valid target tag used: ' % fabric.colors.red(tag_requested) + ) env.tag = tag_requested - env.local_tmp_root_app = os.path.join(env.local_tmp_dir, - '%(application_name)s-%(tag)s' % env) - env.local_tmp_root_app_package = os.path.join(env.local_tmp_root_app, - env.root_package_name) + env.local_tmp_root_app = os.path.join(env.local_tmp_dir, '%(application_name)s-%(tag)s' % env) + env.local_tmp_root_app_package = os.path.join(env.local_tmp_root_app, env.root_package_name) fabric.api.require('tag', provided_by=['tag', 'head']) fabric.api.require('remote_project_dir', provided_by=env.goals) archive_prefix = '%s-%s' % (env.application_name, env.tag.lower()) - tarball = pydiploy.require.git.archive(env.application_name, - prefix='%s/' % archive_prefix, - specific_folder=env.remote_repo_specific_folder if "remote_repo_specific_folder" in env else "", - tag=env.tag, - remote=env.remote_repo_url) + tarball = pydiploy.require.git.archive( + env.application_name, + prefix='%s/' % archive_prefix, + specific_folder=env.remote_repo_specific_folder if "remote_repo_specific_folder" in env else "", + tag=env.tag, + remote=env.remote_repo_url, + ) with fabric.api.lcd(env.local_tmp_dir): # remove existing extracted dir from tarball - if os.path.exists('%s/%s' % (env.local_tmp_dir,archive_prefix)): + if os.path.exists('%s/%s' % (env.local_tmp_dir, archive_prefix)): fabric.api.local('rm -rf %s' % archive_prefix) fabric.api.local('tar xvf %s' % os.path.basename(tarball)) + # add deployed.json file + with open('%s/%s_info.json' % (env.local_tmp_root_app, env.application_name), 'w') as f: + data = {} + data['info'] = [{'tag': env.tag, 'python_version': env.remote_python_version,}] + + json.dump(data, f) if 'run_tests_command' in env and env.run_tests_command: run_tests() # TODO: see if some excluded files / dir # are not in fact usefull in certain projects - exclude_files = ['fabfile', 'MANIFEST.in', '*.ignore', 'docs', - '*.log', 'bin', 'manage.py', '.tox', - '%s/wsgi.py' % env.root_package_name, '*.db', - '.gitignore', '.gitattributes'] - exclude_files += ['%s/settings/%s.py' % (env.root_package_name, goal) - for goal in env.goals] + exclude_files = [ + 'fabfile', + 'MANIFEST.in', + '*.ignore', + 'docs', + '*.log', + 'bin', + 'manage.py', + '.tox', + '%s/wsgi.py' % env.root_package_name, + '*.db', + '.gitignore', + '.gitattributes', + ] + exclude_files += ['%s/settings/%s.py' % (env.root_package_name, goal) for goal in env.goals] if env.has_key('excluded_files'): exclude_files += env.excluded_files if env.has_key('cfg_shared_files'): for cfg_shared_file in env.cfg_shared_files: cfg_present = fabtools.files.is_file( - path='%s/config/%s' % ( - env.remote_shared_path, os.path.basename(cfg_shared_file)), - use_sudo=True) + path='%s/config/%s' % (env.remote_shared_path, os.path.basename(cfg_shared_file)), use_sudo=True + ) if cfg_present is None: - fabtools.files.upload_template('%s/%s/%s' % ( - env.local_tmp_dir, - archive_prefix, - cfg_shared_file - ), - os.path.join( - env.remote_shared_path, 'config'), - use_sudo=True) + fabtools.files.upload_template( + '%s/%s/%s' % (env.local_tmp_dir, archive_prefix, cfg_shared_file), + os.path.join(env.remote_shared_path, 'config'), + use_sudo=True, + ) exclude_files += [cfg_shared_file] @@ -132,25 +148,27 @@ def deploy_code(): exclude_files += [symlink_dir] env.remote_current_release = "%(releases_path)s/%(time).0f" % { - 'releases_path': env.remote_releases_path, 'time': time()} + 'releases_path': env.remote_releases_path, + 'time': time(), + } with warn_only(): - result = fabric.contrib.project.rsync_project(env.remote_current_release, - '%s/%s/' % ( - env.local_tmp_dir, - archive_prefix), - delete=True, - extra_opts='--links --rsync-path="sudo -u %s rsync"' % env.remote_owner, - exclude=exclude_files) - + result = fabric.contrib.project.rsync_project( + env.remote_current_release, + '%s/%s/' % (env.local_tmp_dir, archive_prefix), + delete=True, + extra_opts='--links --rsync-path="sudo -u %s rsync"' % env.remote_owner, + exclude=exclude_files, + ) + # Wrong repository url (git archive is empty) if result.return_code == 23: fabric.api.abort(fabric.colors.red("Unable to use repository, please check repository url !")) fabric.api.sudo( - 'chown -R %(user)s:%(group)s %(project_dir)s' % {'user': env.remote_owner, - 'group': env.remote_group, - 'project_dir': env.remote_current_release}) + 'chown -R %(user)s:%(group)s %(project_dir)s' + % {'user': env.remote_owner, 'group': env.remote_group, 'project_dir': env.remote_current_release} + ) # symlink with new release fabric.api.execute(symlink) # set current directory with new release @@ -171,16 +189,23 @@ def rollback_code(): if "releases" in env: nb_releases = len(env.releases) if nb_releases >= 2: - fabric.api.sudo("rm %(current_path)s; ln -s %(previous_release)s %(current_path)s && rm -rf %(current_release)s" % - {'current_release': env.current_release, 'previous_release': env.previous_release, 'current_path': env.remote_current_path}) + fabric.api.sudo( + "rm %(current_path)s; ln -s %(previous_release)s %(current_path)s && rm -rf %(current_release)s" + % { + 'current_release': env.current_release, + 'previous_release': env.previous_release, + 'current_path': env.remote_current_path, + } + ) # elif nb_releases == 1: elif nb_releases == 1: - fabric.api.puts(fabric.colors.red( - 'No rollback only one release found on remote !')) + fabric.api.puts(fabric.colors.red('No rollback only one release found on remote !')) else: - fabric.api.sudo("rm %(current_path)s && rm -rf %(previous_release)s" % - {'current_path': env.remote_current_path, 'previous_release': env.remote_current_release}) + fabric.api.sudo( + "rm %(current_path)s && rm -rf %(previous_release)s" + % {'current_path': env.remote_current_path, 'previous_release': env.remote_current_release} + ) @do_verbose @@ -190,24 +215,34 @@ def symlink(): """ # TODO : really usefull ? (eg : for php apps ...) - fabric.api.sudo("ln -nfs %(shared_path)s/log %(current_release)s/log" % - {'shared_path': env.remote_shared_path, - 'current_release': env.remote_current_release}) + fabric.api.sudo( + "ln -nfs %(shared_path)s/log %(current_release)s/log" + % {'shared_path': env.remote_shared_path, 'current_release': env.remote_current_release} + ) if env.has_key('cfg_shared_files'): for cfg_shared_file in env.cfg_shared_files: - fabric.api.sudo("ln -nfs %(shared_path)s/config/%(file_name)s %(current_release)s/%(file)s" % - {'shared_path': env.remote_shared_path, - 'current_release': env.remote_current_release, - 'file': cfg_shared_file, - 'file_name': os.path.basename(cfg_shared_file)}) + fabric.api.sudo( + "ln -nfs %(shared_path)s/config/%(file_name)s %(current_release)s/%(file)s" + % { + 'shared_path': env.remote_shared_path, + 'current_release': env.remote_current_release, + 'file': cfg_shared_file, + 'file_name': os.path.basename(cfg_shared_file), + } + ) if env.has_key('extra_symlink_dirs'): for extra_symlink_dir in env.extra_symlink_dirs: - fabric.api.sudo("ln -nfs %(shared_path)s/%(dir_name)s %(current_release)s/%(dir_name)s" % - {'shared_path': env.remote_shared_path, - 'current_release': env.remote_current_release, - 'dir_name': extra_symlink_dir}) + fabric.api.sudo( + "ln -nfs %(shared_path)s/%(dir_name)s %(current_release)s/%(dir_name)s" + % { + 'shared_path': env.remote_shared_path, + 'current_release': env.remote_current_release, + 'dir_name': extra_symlink_dir, + } + ) + @do_verbose def run_tests(): diff --git a/tests/test_require_circus.py b/tests/test_require_circus.py index ca87203..94e1e50 100644 --- a/tests/test_require_circus.py +++ b/tests/test_require_circus.py @@ -5,9 +5,8 @@ from unittest import TestCase from fabric.api import env -from mock import call, Mock, patch -from pydiploy.require.circus import (app_circus_conf, app_reload, circus_pkg, - upstart) +from mock import Mock, call, patch +from pydiploy.require.circus import app_circus_conf, app_reload, circus_pkg, upstart class CircusCheck(TestCase): @@ -35,7 +34,9 @@ def tearDown(self): @patch('fabtools.require.python.install', return_value=Mock()) @patch('fabtools.files.upload_template', return_value=Mock()) @patch('fabtools.require.files.directory', return_value=Mock()) - def test_circus_pkg_old(self, files_directory, upload_template, python_install, deb_ppa, deb_packages, distrib_release, distrib_id): + def test_circus_pkg_old( + self, files_directory, upload_template, python_install, deb_ppa, deb_packages, distrib_release, distrib_id + ): # no_circus_web = True env.no_circus_web = True @@ -45,8 +46,7 @@ def test_circus_pkg_old(self, files_directory, upload_template, python_install, self.assertTrue(distrib_id.called) self.assertTrue(deb_packages.called) - self.assertEqual( - deb_packages.call_args_list, [call(['libzmq-dev', 'libevent-dev'], update=False)]) + self.assertEqual(deb_packages.call_args_list, [call(['libzmq-dev', 'libevent-dev'], update=False)]) # self.assertTrue(deb_ppa.called) # self.assertEqual(deb_ppa.call_args_list, [ @@ -54,29 +54,26 @@ def test_circus_pkg_old(self, files_directory, upload_template, python_install, self.assertTrue(python_install.called) - self.assertEqual( - python_install.call_args_list, [call('circus', upgrade=False, use_sudo=True)]) + self.assertEqual(python_install.call_args_list, [call('circus', upgrade=False, use_sudo=True)]) self.assertTrue(upload_template.called) - self.assertTrue( - str(upload_template.call_args).find("'circus.ini.tpl'") > 0) - self.assertTrue( - str(upload_template.call_args).find("'remote_home/.circus.ini'") > 0) - self.assertTrue(str(upload_template.call_args) - .find("template_dir='lib_path/templates'") > 0) - self.assertTrue( - str(upload_template.call_args).find("user='remote_owner'") > 0) + self.assertTrue(str(upload_template.call_args).find("'circus.ini.tpl'") > 0) + self.assertTrue(str(upload_template.call_args).find("'remote_home/.circus.ini'") > 0) + self.assertTrue(str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0) + self.assertTrue(str(upload_template.call_args).find("user='remote_owner'") > 0) self.assertTrue(files_directory.called) - self.assertEqual(files_directory.call_args, - call(owner='remote_owner', path='remote_home/.circus.d', use_sudo=True, group='remote_group', mode='750')) + self.assertEqual( + files_directory.call_args, + call(owner='remote_owner', path='remote_home/.circus.d', use_sudo=True, group='remote_group', mode='750'), + ) # test circus_package_name env.circus_package_name = "https://github.com/githubaccount/circus/archive/master.zip" circus_pkg() self.assertEqual( - python_install.call_args_list[1], call( - 'https://github.com/githubaccount/circus/archive/master.zip', upgrade=False, use_sudo=True) + python_install.call_args_list[1], + call('https://github.com/githubaccount/circus/archive/master.zip', upgrade=False, use_sudo=True), ) # test no_circus_web @@ -84,18 +81,26 @@ def test_circus_pkg_old(self, files_directory, upload_template, python_install, del env['circus_package_name'] circus_pkg() self.assertEqual( - python_install.call_args_list[-3:], [call('circus', use_sudo=True, upgrade=False), - call('circus-web', use_sudo=True, upgrade=False), - call('gevent', use_sudo=True, upgrade=False)]) + python_install.call_args_list[-3:], + [ + call('circus', use_sudo=True, upgrade=False), + call('circus-web', use_sudo=True, upgrade=False), + call('gevent', use_sudo=True, upgrade=False), + ], + ) # test circus_backend env.circus_backend = "test_circus_backend" circus_pkg() self.assertEqual( - python_install.call_args_list[-4:], [call('circus', use_sudo=True, upgrade=False), - call('circus-web', use_sudo=True, upgrade=False), - call('gevent', use_sudo=True, upgrade=False), - call('test_circus_backend', use_sudo=True, upgrade=False)]) + python_install.call_args_list[-4:], + [ + call('circus', use_sudo=True, upgrade=False), + call('circus-web', use_sudo=True, upgrade=False), + call('gevent', use_sudo=True, upgrade=False), + call('test_circus_backend', use_sudo=True, upgrade=False), + ], + ) @patch('fabtools.system.distrib_id', return_value='Ubuntu') @patch('fabtools.system.distrib_release', return_value='18.04') @@ -104,7 +109,9 @@ def test_circus_pkg_old(self, files_directory, upload_template, python_install, @patch('fabtools.require.python.install', return_value=Mock()) @patch('fabtools.files.upload_template', return_value=Mock()) @patch('fabtools.require.files.directory', return_value=Mock()) - def test_circus_pkg(self, files_directory, upload_template, python_install, deb_ppa, deb_packages, distrib_release, distrib_id): + def test_circus_pkg( + self, files_directory, upload_template, python_install, deb_ppa, deb_packages, distrib_release, distrib_id + ): # no_circus_web = True env.no_circus_web = True @@ -114,8 +121,7 @@ def test_circus_pkg(self, files_directory, upload_template, python_install, deb_ self.assertTrue(distrib_id.called) self.assertTrue(deb_packages.called) - self.assertEqual( - deb_packages.call_args_list, [call(['libzmq3-dev', 'libevent-dev'], update=False)]) + self.assertEqual(deb_packages.call_args_list, [call(['libzmq3-dev', 'libevent-dev'], update=False)]) # self.assertTrue(deb_ppa.called) # self.assertEqual(deb_ppa.call_args_list, [ @@ -123,29 +129,26 @@ def test_circus_pkg(self, files_directory, upload_template, python_install, deb_ self.assertTrue(python_install.called) - self.assertEqual( - python_install.call_args_list, [call('circus', upgrade=False, use_sudo=True)]) + self.assertEqual(python_install.call_args_list, [call('circus', upgrade=False, use_sudo=True)]) self.assertTrue(upload_template.called) - self.assertTrue( - str(upload_template.call_args).find("'circus.ini.tpl'") > 0) - self.assertTrue( - str(upload_template.call_args).find("'remote_home/.circus.ini'") > 0) - self.assertTrue(str(upload_template.call_args) - .find("template_dir='lib_path/templates'") > 0) - self.assertTrue( - str(upload_template.call_args).find("user='remote_owner'") > 0) + self.assertTrue(str(upload_template.call_args).find("'circus.ini.tpl'") > 0) + self.assertTrue(str(upload_template.call_args).find("'remote_home/.circus.ini'") > 0) + self.assertTrue(str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0) + self.assertTrue(str(upload_template.call_args).find("user='remote_owner'") > 0) self.assertTrue(files_directory.called) - self.assertEqual(files_directory.call_args, - call(owner='remote_owner', path='remote_home/.circus.d', use_sudo=True, group='remote_group', mode='750')) + self.assertEqual( + files_directory.call_args, + call(owner='remote_owner', path='remote_home/.circus.d', use_sudo=True, group='remote_group', mode='750'), + ) # test circus_package_name env.circus_package_name = "https://github.com/githubaccount/circus/archive/master.zip" circus_pkg() self.assertEqual( - python_install.call_args_list[1], call( - 'https://github.com/githubaccount/circus/archive/master.zip', upgrade=False, use_sudo=True) + python_install.call_args_list[1], + call('https://github.com/githubaccount/circus/archive/master.zip', upgrade=False, use_sudo=True), ) # test no_circus_web @@ -153,32 +156,36 @@ def test_circus_pkg(self, files_directory, upload_template, python_install, deb_ del env['circus_package_name'] circus_pkg() self.assertEqual( - python_install.call_args_list[-3:], [call('circus', use_sudo=True, upgrade=False), - call('circus-web', use_sudo=True, upgrade=False), - call('gevent', use_sudo=True, upgrade=False)]) + python_install.call_args_list[-3:], + [ + call('circus', use_sudo=True, upgrade=False), + call('circus-web', use_sudo=True, upgrade=False), + call('gevent', use_sudo=True, upgrade=False), + ], + ) # test circus_backend env.circus_backend = "test_circus_backend" circus_pkg() self.assertEqual( - python_install.call_args_list[-4:], [call('circus', use_sudo=True, upgrade=False), - call('circus-web', use_sudo=True, upgrade=False), - call('gevent', use_sudo=True, upgrade=False), - call('test_circus_backend', use_sudo=True, upgrade=False)]) + python_install.call_args_list[-4:], + [ + call('circus', use_sudo=True, upgrade=False), + call('circus-web', use_sudo=True, upgrade=False), + call('gevent', use_sudo=True, upgrade=False), + call('test_circus_backend', use_sudo=True, upgrade=False), + ], + ) @patch('fabtools.files.upload_template', return_value=Mock()) def test_app_circus_conf(self, upload_template): app_circus_conf() self.assertTrue(upload_template.called) - self.assertTrue( - str(upload_template.call_args).find("'app.ini.tpl'") > 0) - self.assertTrue(str(upload_template.call_args).find( - "'remote_home/.circus.d/application_name.ini'") > 0) - self.assertTrue(str(upload_template.call_args) - .find("template_dir='lib_path/templates'") > 0) - self.assertTrue( - str(upload_template.call_args).find("user='remote_owner'") > 0) + self.assertTrue(str(upload_template.call_args).find("'app.ini.tpl'") > 0) + self.assertTrue(str(upload_template.call_args).find("'remote_home/.circus.d/application_name.ini'") > 0) + self.assertTrue(str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0) + self.assertTrue(str(upload_template.call_args).find("user='remote_owner'") > 0) @patch('fabtools.files.is_dir', return_value=False) @patch('fabtools.files.upload_template', return_value=Mock()) @@ -186,12 +193,9 @@ def test_upstart(self, upload_template, is_systemd): upstart() self.assertTrue(upload_template.called) - self.assertTrue( - str(upload_template.call_args).find("'upstart.conf.tpl'") > 0) - self.assertTrue( - str(upload_template.call_args).find("'/etc/init/circus.conf'") > 0) - self.assertTrue(str(upload_template.call_args) - .find("template_dir='lib_path/templates'") > 0) + self.assertTrue(str(upload_template.call_args).find("'upstart.conf.tpl'") > 0) + self.assertTrue(str(upload_template.call_args).find("'/etc/init/circus.conf'") > 0) + self.assertTrue(str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0) self.assertTrue(str(upload_template.call_args).find("user='root'") > 0) @patch('fabric.api.sudo', return_value=Mock()) @@ -201,12 +205,9 @@ def test_upstart_systemd(self, upload_template, is_systemd, mock_reload_daemon): upstart() self.assertTrue(upload_template.called) - self.assertTrue( - str(upload_template.call_args).find("'circus.service.tpl'") > 0) - self.assertTrue( - str(upload_template.call_args).find("'/etc/systemd/system/circus.service'") > 0) - self.assertTrue(str(upload_template.call_args) - .find("template_dir='lib_path/templates'") > 0) + self.assertTrue(str(upload_template.call_args).find("'circus.service.tpl'") > 0) + self.assertTrue(str(upload_template.call_args).find("'/etc/systemd/system/circus.service'") > 0) + self.assertTrue(str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0) self.assertTrue(str(upload_template.call_args).find("user='root'") > 0) self.assertTrue(mock_reload_daemon.called) @@ -222,14 +223,14 @@ def test_app_reload(self, api_sudo, api_settings, is_systemd): app_reload() self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args_list, [call('status circus'), call( - 'circusctl reloadconfig'), call('circusctl restart application_name')]) + self.assertEqual( + api_sudo.call_args_list, [call('status circus'), call('circusctl reloadconfig')], + ) self.assertTrue(api_settings.called) - self.assertEqual( - api_settings.call_args, call(sudo_user='remote_owner')) + self.assertEqual(api_settings.call_args, call(sudo_user='remote_owner')) - # test if not running + #  test if not running api_sudo.return_value = 'stopped' api_sudo.called = False api_settings.called = False @@ -252,14 +253,19 @@ def test_app_reload_systemd(self, api_sudo, api_settings, is_systemd): app_reload() self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args_list, [call('systemctl is-active circus.service'), call( - 'circusctl reloadconfig'), call('circusctl restart application_name')]) + self.assertEqual( + api_sudo.call_args_list, + [ + call('systemctl is-active circus.service'), + call('circusctl reloadconfig'), + # call('circusctl restart application_name'), + ], + ) self.assertTrue(api_settings.called) - self.assertEqual( - api_settings.call_args, call(sudo_user='remote_owner')) + self.assertEqual(api_settings.call_args, call(sudo_user='remote_owner')) - # test if not running + #  test if not running api_sudo.return_value = 'inactive' api_sudo.called = False api_settings.called = False diff --git a/tests/test_require_system.py b/tests/test_require_system.py index f39d377..8dbbb07 100644 --- a/tests/test_require_system.py +++ b/tests/test_require_system.py @@ -5,12 +5,20 @@ from unittest import TestCase from fabric.api import env -from mock import call, Mock, patch -from pydiploy.require.system import (check_python3_install, add_group, - add_user, install_extra_packages, - install_extra_ppa, install_extra_source, - package_installed, permissions, set_locale, - set_timezone, update_pkg_index) +from mock import Mock, call, patch +from pydiploy.require.system import ( + add_group, + add_user, + check_python3_install, + install_extra_packages, + install_extra_ppa, + install_extra_source, + package_installed, + permissions, + set_locale, + set_timezone, + update_pkg_index, +) class SystemCheck(TestCase): @@ -39,12 +47,16 @@ def test_add_user(self, require_group, require_user, require_sudoer): add_user(commands='mycommand') self.assertTrue(require_group.called) - self.assertEqual(require_group.call_args, call( - '%remote_group', passwd=False, commands='mycommand', operators='remote_owner,root')) + self.assertEqual( + require_group.call_args, + call('%remote_group', passwd=False, commands='mycommand', operators='remote_owner,root'), + ) self.assertTrue(require_user.called) - self.assertEqual(require_user.call_args, call( - 'remote_owner', create_group=True, shell='/bin/bash', group='remote_group', create_home=True)) + self.assertEqual( + require_user.call_args, + call('remote_owner', create_group=True, shell='/bin/bash', group='remote_group', create_home=True), + ) self.assertTrue(require_sudoer.called) self.assertEqual(require_sudoer.call_args, call('remote_group')) @@ -75,51 +87,49 @@ def test_set_locale(self, api_sudo, api_run): self.assertTrue(api_run.called) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args_list, [ - call('locale-gen FR_fr'), call('/usr/sbin/update-locale LANG=FR_fr')]) + self.assertEqual( + api_sudo.call_args_list, [call('locale-gen FR_fr'), call('/usr/sbin/update-locale LANG=FR_fr')] + ) @patch('fabtools.files.is_link', return_value=False) @patch('fabtools.system.distrib_id', return_value='Notsupported') @patch('fabric.api.sudo', return_value=Mock()) def test_set_timezone(self, api_sudo, distrib_id, is_link): set_timezone() - # test error + #  test error self.assertTrue(distrib_id.called) self.assertFalse(api_sudo.called) - # test if it works + #  test if it works distrib_id.return_value = 'Ubuntu' set_timezone() self.assertTrue(distrib_id.called) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, call( - 'cp -f /usr/share/zoneinfo/mytimezone /etc/localtime')) + self.assertEqual(api_sudo.call_args, call('cp -f /usr/share/zoneinfo/mytimezone /etc/localtime')) @patch('fabtools.files.is_link', return_value=True) @patch('fabtools.system.distrib_id', return_value='Notsupported') @patch('fabric.api.sudo', return_value=Mock()) def test_set_timezone_link(self, api_sudo, distrib_id, is_link): set_timezone() - # test error + #  test error self.assertTrue(distrib_id.called) self.assertFalse(api_sudo.called) - # test if it works + #  test if it works distrib_id.return_value = 'Ubuntu' set_timezone() self.assertTrue(distrib_id.called) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, call( - 'ln -sf /usr/share/zoneinfo/mytimezone /etc/localtime')) + self.assertEqual(api_sudo.call_args, call('ln -sf /usr/share/zoneinfo/mytimezone /etc/localtime')) @patch('fabric.api.sudo', return_value=Mock()) def test_permissions(self, api_sudo): permissions() self.assertTrue(api_sudo.called) - self.assertEqual( - api_sudo.call_args, call('chmod -R g+w remote_project_dir')) + self.assertEqual(api_sudo.call_args, call('chmod -R g+w remote_project_dir')) @patch('fabric.api.settings', return_value=Mock()) @patch('fabric.api.sudo', return_value=Mock()) @@ -130,8 +140,7 @@ def test_package_installed(self, api_sudo, api_settings): package_installed("package") self.assertTrue(api_sudo.called) - self.assertEqual( - api_sudo.call_args, call('dpkg-query -l "package" | grep -q ^.i')) + self.assertEqual(api_sudo.call_args, call('dpkg-query -l "package" | grep -q ^.i')) @patch('fabtools.require.deb.packages', return_value=Mock()) def test_install_extra_packages(self, deb_pkgs): @@ -143,27 +152,37 @@ def test_install_extra_packages(self, deb_pkgs): @patch('fabtools.require.deb.ppa', return_value=Mock()) def test_install_extra_ppa(self, deb_ppa): - install_extra_ppa(['ppa:myppa/ppa', ]) + install_extra_ppa( + ['ppa:myppa/ppa',] + ) self.assertTrue(deb_ppa.called) self.assertEqual(deb_ppa.call_args, call('ppa:myppa/ppa')) @patch('fabtools.require.deb.source', return_value=Mock()) def test_install_extra_source(self, deb_source): - install_extra_source([['mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'],]) + install_extra_source( + [['mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'],] + ) self.assertTrue(deb_source.called) - self.assertEqual(deb_source.call_args, call('mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen')) + self.assertEqual( + deb_source.call_args, + call('mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'), + ) @patch('fabtools.require.deb.package', return_value=Mock()) @patch('fabtools.require.deb.source', return_value=Mock()) @patch('fabtools.require.deb.ppa', return_value=Mock()) + @patch('fabric.api.sudo', return_value=Mock()) @patch('fabtools.require.deb.packages', return_value=Mock()) @patch('fabtools.system.distrib_release', return_value='13') @patch('fabtools.system.distrib_id', return_value='Ubuntu') @patch('pydiploy.require.system.package_installed', return_value=False) - def test_check_python3_install(self, pkg_installed, sys_distrib, sys_distrib_id, deb_pkgs, deb_ppa, deb_source, deb_pkg): + def test_check_python3_install( + self, pkg_installed, sys_distrib, sys_distrib_id, deb_pkgs, fab_sudo, deb_ppa, deb_source, deb_pkg + ): check_python3_install() - self.assertTrue(deb_pkgs.called) - self.assertTrue(deb_ppa.called) self.assertTrue(deb_pkg.called) + # self.assertTrue(deb_ppa.called) + # self.assertTrue(deb_pkgs.called) From 9f9ad55866c0bd067c23f5c6e07fcccc8bca0a3c Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Thu, 4 Jun 2020 17:51:43 +0200 Subject: [PATCH 41/44] Fix tests --- tests/test_require_python.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/tests/test_require_python.py b/tests/test_require_python.py index f9adfae..ba0161b 100644 --- a/tests/test_require_python.py +++ b/tests/test_require_python.py @@ -5,7 +5,7 @@ from unittest import TestCase from fabric.api import env -from mock import call, Mock, patch +from mock import Mock, call, patch from pydiploy.require.python.utils import application_dependencies, python_pkg from pydiploy.require.python.virtualenv import virtualenv @@ -31,14 +31,14 @@ def tearDown(self): @patch('fabtools.require.deb.packages', return_value=Mock()) @patch('fabtools.require.python.install', return_value=Mock()) - def test_python_pkg(self, python_install, deb_packages): + @patch('fabtools.system.distrib_release', return_value='13') + @patch('fabtools.system.distrib_id', return_value='Ubuntu') + def test_python_pkg(self, dist_id, dist_rls, python_install, deb_packages): python_pkg() self.assertTrue(deb_packages.called) - self.assertEqual(deb_packages.call_args, call( - ['python-dev', 'python-pip'], update=False)) + self.assertEqual(deb_packages.call_args, call(['python-dev', 'python-pip'], update=False)) self.assertTrue(python_install.called) - self.assertEqual( - python_install.call_args, call('pip', upgrade=True, use_sudo=True)) + self.assertEqual(python_install.call_args, call('pip', upgrade=True, use_sudo=True)) @patch('fabtools.python.virtualenv', return_value=Mock()) @patch('fabric.api.cd', return_value=Mock()) @@ -59,8 +59,7 @@ def test_application_dependencies(self, install_requirements, api_sudo, python_i self.assertEqual(api_cd.call_args, call('remote_current_path')) self.assertTrue(python_virtualenv.called) - self.assertEqual( - python_virtualenv.call_args, call('remote_virtualenv_dir')) + self.assertEqual(python_virtualenv.call_args, call('remote_virtualenv_dir')) # test oracle @@ -95,9 +94,19 @@ def test_virtualenv(self, python_virtualenv, files_directory): virtualenv() self.assertTrue(python_virtualenv.called) - self.assertEqual(python_virtualenv.call_args, - call('remote_virtualenv_dir', clear=False, use_sudo=True, venv_python='/usr/bin/python2.7', user='remote_owner')) + self.assertEqual( + python_virtualenv.call_args, + call( + 'remote_virtualenv_dir', + clear=False, + use_sudo=True, + venv_python='/usr/bin/python2.7', + user='remote_owner', + ), + ) self.assertTrue(files_directory.called) - self.assertEqual(files_directory.call_args, call( - 'remote_virtualenv_dir', owner='remote_owner', use_sudo=True, group='remote_group')) + self.assertEqual( + files_directory.call_args, + call('remote_virtualenv_dir', owner='remote_owner', use_sudo=True, group='remote_group'), + ) From 191ea3437fa1175e5db23f868a5b37c2f101f2eb Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Thu, 4 Jun 2020 17:58:21 +0200 Subject: [PATCH 42/44] Enhance tox.ini config --- tox.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 42495e7..0ba2a00 100644 --- a/tox.ini +++ b/tox.ini @@ -6,5 +6,8 @@ deps= coverage mock pytest -commands=coverage run -m py.test -rxs +commands= + coverage erase + coverage run -m py.test -rxs + coverage report From 2e9668f70eecf1cf23d609d7cca337725ec036e9 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Fri, 5 Jun 2020 10:09:51 +0200 Subject: [PATCH 43/44] Return to 88 chars/line & more --- .gitignore | 1 + pydiploy/bottle.py | 57 ++-- pydiploy/decorators.py | 17 +- pydiploy/django.py | 51 ++-- pydiploy/php.py | 38 ++- pydiploy/prepare.py | 82 ++++-- pydiploy/require/apache.py | 84 +++--- pydiploy/require/bottle/command.py | 7 +- pydiploy/require/bottle/utils.py | 73 ++--- pydiploy/require/circus.py | 23 +- pydiploy/require/databases/ldap.py | 4 +- pydiploy/require/databases/mongodb.py | 13 +- pydiploy/require/databases/mysql.py | 18 +- pydiploy/require/databases/oracle.py | 39 ++- pydiploy/require/databases/postgres.py | 51 +++- pydiploy/require/databases/sap.py | 16 +- pydiploy/require/databases/sqlite.py | 4 +- pydiploy/require/django/command.py | 19 +- pydiploy/require/django/utils.py | 81 +++--- pydiploy/require/git.py | 15 +- pydiploy/require/nginx.py | 111 ++++---- pydiploy/require/python/utils.py | 36 ++- pydiploy/require/python/virtualenv.py | 21 +- pydiploy/require/releases_manager.py | 91 ++++-- pydiploy/require/system.py | 4 +- pydiploy/scripts.py | 15 +- pydiploy/simple.py | 33 ++- pydiploy/version.py | 2 +- tests/functional_tests/conftest.py | 18 +- .../test_pre_install_backend.py | 44 ++- .../test_pre_install_frontend.py | 17 +- tests/test_bottle.py | 242 +++++++++++----- tests/test_django.py | 260 ++++++++++++------ tests/test_prepare.py | 123 +++++---- tests/test_releases_manager.py | 199 ++++++++++---- tests/test_require_bottle.py | 53 ++-- tests/test_require_circus.py | 121 ++++++-- tests/test_require_databases.py | 234 +++++++++------- tests/test_require_django.py | 168 ++++++----- tests/test_require_git.py | 44 ++- tests/test_require_nginx.py | 173 +++++++----- tests/test_require_python.py | 19 +- tests/test_require_simple.py | 53 ++-- tests/test_require_system.py | 58 +++- tests/test_scripts.py | 5 +- tests/test_simple.py | 124 ++++++--- tools/pydiployfabfile.sublime-snippet | 61 ++-- tools/pydiployfabfilebottle.sublime-snippet | 47 ++-- tools/pydiployfabfilesimple.sublime-snippet | 35 ++- 49 files changed, 2023 insertions(+), 1081 deletions(-) diff --git a/.gitignore b/.gitignore index dcbd41f..d951527 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,4 @@ docs/_build htmlcov .cache .ropeproject +.vscode diff --git a/pydiploy/bottle.py b/pydiploy/bottle.py index 29c47bb..9ac514e 100644 --- a/pydiploy/bottle.py +++ b/pydiploy/bottle.py @@ -14,8 +14,9 @@ def wrap_deploy(): yield except SystemExit: fabric.api.execute(rollback) - fabric.api.abort(fabric.colors.red( - "Deploy failed rollbacking process launched")) + fabric.api.abort( + fabric.colors.red("Deploy failed rollbacking process launched") + ) @do_verbose @@ -23,21 +24,23 @@ def application_packages(update=False): """ Installs all packages for the app """ fabtools.require.deb.packages(['gettext'], update=update) if env.remote_python_version >= 3: - fabric.api.execute(pydiploy.require.system.check_python3_install, - version='python%s' % env.remote_python_version) + fabric.api.execute( + pydiploy.require.system.check_python3_install, + version='python%s' % env.remote_python_version, + ) fabric.api.execute(pydiploy.require.python.utils.python_pkg) if 'extra_ppa_to_install' in env: fabric.api.execute( - pydiploy.require.system.install_extra_ppa, - env.extra_ppa_to_install) + pydiploy.require.system.install_extra_ppa, env.extra_ppa_to_install + ) if 'extra_source_to_install' in env: fabric.api.execute( - pydiploy.require.system.install_extra_source, - env.extra_source_to_install) + pydiploy.require.system.install_extra_source, env.extra_source_to_install + ) if 'extra_pkg_to_install' in env: fabric.api.execute( - pydiploy.require.system.install_extra_packages, - env.extra_pkg_to_install) + pydiploy.require.system.install_extra_packages, env.extra_pkg_to_install + ) def pre_install_backend(commands='/usr/bin/rsync', upgrade_circus=False): @@ -47,8 +50,7 @@ def pre_install_backend(commands='/usr/bin/rsync', upgrade_circus=False): fabric.api.execute(pydiploy.require.system.set_timezone) fabric.api.execute(pydiploy.require.system.update_pkg_index) fabric.api.execute(application_packages) - fabric.api.execute(pydiploy.require.circus.circus_pkg, - update=upgrade_circus) + fabric.api.execute(pydiploy.require.circus.circus_pkg, update=upgrade_circus) fabric.api.execute(pydiploy.require.python.virtualenv.virtualenv) fabric.api.execute(pydiploy.require.circus.upstart) @@ -67,11 +69,10 @@ def deploy_backend(upgrade_pkg=False, **kwargs): fabric.api.execute(pydiploy.require.releases_manager.deploy_code) fabric.api.execute(pydiploy.require.django.utils.deploy_wsgi_file) fabric.api.execute( - pydiploy.require.python.utils.application_dependencies, - upgrade_pkg) + pydiploy.require.python.utils.application_dependencies, upgrade_pkg + ) # TODO PUT THIS METHOD IN OTHER PACKAGE - fabric.api.execute(pydiploy.require.bottle.utils.app_settings, - **kwargs) + fabric.api.execute(pydiploy.require.bottle.utils.app_settings, **kwargs) fabric.api.execute(pydiploy.require.bottle.utils.deploy_environ_file) fabric.api.execute(pydiploy.require.bottle.command.bottle_prepare) fabric.api.execute(pydiploy.require.system.permissions) @@ -122,7 +123,7 @@ def set_app_up(): fabric.api.execute(pydiploy.require.nginx.set_website_up) -def install_postgres_server(user=None,dbname=None,password=None): +def install_postgres_server(user=None, dbname=None, password=None): """ Install postgres server & add user for postgres if no parameters are provided using (if exists) :: @@ -134,16 +135,30 @@ def install_postgres_server(user=None,dbname=None,password=None): """ if not (user and dbname and password): - if all([e in env.keys() for e in ('default_db_user', 'default_db_name', 'default_db_password')]): + if all( + [ + e in env.keys() + for e in ('default_db_user', 'default_db_name', 'default_db_password') + ] + ): user = env.default_db_user dbname = env.default_db_name password = env.default_db_password else: - fabric.api.abort('Please provide user,dbname,password parameters for postgres.') + fabric.api.abort( + 'Please provide user,dbname,password parameters for postgres.' + ) fabric.api.execute(pydiploy.require.databases.postgres.install_postgres_server) - fabric.api.execute(pydiploy.require.databases.postgres.add_postgres_user,user,password=password) - fabric.api.execute(pydiploy.require.databases.postgres.add_postgres_database,dbname,owner=user,locale=env.locale) + fabric.api.execute( + pydiploy.require.databases.postgres.add_postgres_user, user, password=password + ) + fabric.api.execute( + pydiploy.require.databases.postgres.add_postgres_database, + dbname, + owner=user, + locale=env.locale, + ) def install_oracle_client(): diff --git a/pydiploy/decorators.py b/pydiploy/decorators.py index 49f8dd5..1fa06e2 100644 --- a/pydiploy/decorators.py +++ b/pydiploy/decorators.py @@ -4,6 +4,7 @@ import fabric from fabric.api import env + """ Decorators @@ -18,11 +19,19 @@ def do_verbose(view_func): @wraps(view_func) def wrapper(*args, **kwargs): - custom_hide = ['running', 'stdout', 'stderr'] if 'verbose_output' in\ - env and not env.verbose_output else [] + custom_hide = ( + ['running', 'stdout', 'stderr'] + if 'verbose_output' in env and not env.verbose_output + else [] + ) with fabric.context_managers.hide(*custom_hide): - fabric.api.puts("%s : %s" % (view_func.__name__, fabric.colors.green("Executing"))) + fabric.api.puts( + "%s : %s" % (view_func.__name__, fabric.colors.green("Executing")) + ) res = view_func(*args, **kwargs) - fabric.api.puts("%s : %s" % (view_func.__name__, fabric.colors.green("Done"))) + fabric.api.puts( + "%s : %s" % (view_func.__name__, fabric.colors.green("Done")) + ) return res + return wrapper diff --git a/pydiploy/django.py b/pydiploy/django.py index 6b881ad..b2b3c54 100644 --- a/pydiploy/django.py +++ b/pydiploy/django.py @@ -22,8 +22,9 @@ def wrap_deploy(): yield except SystemExit: fabric.api.execute(rollback) - fabric.api.abort(fabric.colors.red( - "Deploy failed rollbacking process launched")) + fabric.api.abort( + fabric.colors.red("Deploy failed rollbacking process launched") + ) @do_verbose @@ -32,18 +33,23 @@ def application_packages(update=False): fabtools.require.deb.packages(['gettext'], update=update) if env.remote_python_version >= 3: - fabric.api.execute(pydiploy.require.system.check_python3_install, - version='python%s' % env.remote_python_version) + fabric.api.execute( + pydiploy.require.system.check_python3_install, + version='python%s' % env.remote_python_version, + ) fabric.api.execute(pydiploy.require.python.utils.python_pkg) if env.has_key('extra_ppa_to_install'): fabric.api.execute( - pydiploy.require.system.install_extra_ppa, env.extra_ppa_to_install) + pydiploy.require.system.install_extra_ppa, env.extra_ppa_to_install + ) if env.has_key('extra_source_to_install'): fabric.api.execute( - pydiploy.require.system.install_extra_source, env.extra_source_to_install) + pydiploy.require.system.install_extra_source, env.extra_source_to_install + ) if env.has_key('extra_pkg_to_install'): fabric.api.execute( - pydiploy.require.system.install_extra_packages, env.extra_pkg_to_install) + pydiploy.require.system.install_extra_packages, env.extra_pkg_to_install + ) def pre_install_backend(commands='/usr/bin/rsync', upgrade_circus=False): @@ -73,10 +79,9 @@ def deploy_backend(upgrade_pkg=False, **kwargs): fabric.api.execute(pydiploy.require.django.utils.deploy_manage_file) fabric.api.execute(pydiploy.require.django.utils.deploy_wsgi_file) fabric.api.execute( - pydiploy.require.python.utils.application_dependencies, - upgrade_pkg) - fabric.api.execute(pydiploy.require.django.utils.app_settings, - **kwargs) + pydiploy.require.python.utils.application_dependencies, upgrade_pkg + ) + fabric.api.execute(pydiploy.require.django.utils.app_settings, **kwargs) fabric.api.execute(pydiploy.require.django.command.django_prepare) fabric.api.execute(pydiploy.require.system.permissions) fabric.api.execute(pydiploy.require.circus.app_reload) @@ -136,7 +141,7 @@ def custom_manage_command(cmd): fabric.api.execute(pydiploy.require.django.command.django_custom_cmd, cmd) -def install_postgres_server(user=None,dbname=None,password=None): +def install_postgres_server(user=None, dbname=None, password=None): """ Install postgres server & add user for postgres if no parameters are provided using (if exists) :: @@ -148,16 +153,30 @@ def install_postgres_server(user=None,dbname=None,password=None): """ if not (user and dbname and password): - if all([e in env.keys() for e in ('default_db_user', 'default_db_name', 'default_db_password')]): + if all( + [ + e in env.keys() + for e in ('default_db_user', 'default_db_name', 'default_db_password') + ] + ): user = env.default_db_user dbname = env.default_db_name password = env.default_db_password else: - fabric.api.abort('Please provide user,dbname,password parameters for postgres.') + fabric.api.abort( + 'Please provide user,dbname,password parameters for postgres.' + ) fabric.api.execute(pydiploy.require.databases.postgres.install_postgres_server) - fabric.api.execute(pydiploy.require.databases.postgres.add_postgres_user,user,password=password) - fabric.api.execute(pydiploy.require.databases.postgres.add_postgres_database,dbname,owner=user,locale=env.locale) + fabric.api.execute( + pydiploy.require.databases.postgres.add_postgres_user, user, password=password + ) + fabric.api.execute( + pydiploy.require.databases.postgres.add_postgres_database, + dbname, + owner=user, + locale=env.locale, + ) def install_oracle_client(): diff --git a/pydiploy/php.py b/pydiploy/php.py index 6a053ba..919f574 100644 --- a/pydiploy/php.py +++ b/pydiploy/php.py @@ -22,8 +22,9 @@ def wrap_deploy(): yield except SystemExit: fabric.api.execute(rollback) - fabric.api.abort(fabric.colors.red( - "Deploy failed rollbacking process launched")) + fabric.api.abort( + fabric.colors.red("Deploy failed rollbacking process launched") + ) @do_verbose @@ -31,10 +32,12 @@ def application_packages(update=False): """ Installs all packages for php webapp """ if env.has_key('extra_ppa_to_install'): fabric.api.execute( - pydiploy.require.system.install_extra_ppa, env.extra_ppa_to_install) + pydiploy.require.system.install_extra_ppa, env.extra_ppa_to_install + ) if env.has_key('extra_pkg_to_install'): fabric.api.execute( - pydiploy.require.system.install_extra_packages, env.extra_pkg_to_install) + pydiploy.require.system.install_extra_packages, env.extra_pkg_to_install + ) def pre_install_backend(commands='/usr/bin/rsync'): @@ -44,7 +47,7 @@ def pre_install_backend(commands='/usr/bin/rsync'): fabric.api.execute(pydiploy.require.system.set_timezone) fabric.api.execute(pydiploy.require.system.update_pkg_index) fabric.api.execute(pydiploy.require.apache.apache_pkg) - fabtools.require.deb.packages(['php5','libapache2-mod-php5'], update=True) + fabtools.require.deb.packages(['php5', 'libapache2-mod-php5'], update=True) fabric.api.execute(application_packages) @@ -56,6 +59,7 @@ def deploy_backend(upgrade_pkg=False, **kwargs): fabric.api.execute(pydiploy.require.system.permissions) fabric.api.execute(pydiploy.require.releases_manager.cleanup) + def post_install_backend(): fabric.api.execute(pydiploy.require.apache.web_configuration) fabric.api.execute(pydiploy.require.apache.apache_restart) @@ -82,7 +86,7 @@ def set_app_up(): fabric.api.execute(pydiploy.require.apache.set_website_up) -def install_postgres_server(user=None,dbname=None,password=None): +def install_postgres_server(user=None, dbname=None, password=None): """ Install postgres server & add user for postgres if no parameters are provided using (if exists) :: @@ -94,16 +98,30 @@ def install_postgres_server(user=None,dbname=None,password=None): """ if not (user and dbname and password): - if all([e in env.keys() for e in ('default_db_user', 'default_db_name', 'default_db_password')]): + if all( + [ + e in env.keys() + for e in ('default_db_user', 'default_db_name', 'default_db_password') + ] + ): user = env.default_db_user dbname = env.default_db_name password = env.default_db_password else: - fabric.api.abort('Please provide user,dbname,password parameters for postgres.') + fabric.api.abort( + 'Please provide user,dbname,password parameters for postgres.' + ) fabric.api.execute(pydiploy.require.databases.postgres.install_postgres_server) - fabric.api.execute(pydiploy.require.databases.postgres.add_postgres_user,user,password=password) - fabric.api.execute(pydiploy.require.databases.postgres.add_postgres_database,dbname,owner=user,locale=env.locale) + fabric.api.execute( + pydiploy.require.databases.postgres.add_postgres_user, user, password=password + ) + fabric.api.execute( + pydiploy.require.databases.postgres.add_postgres_database, + dbname, + owner=user, + locale=env.locale, + ) def install_oracle_client(): diff --git a/pydiploy/prepare.py b/pydiploy/prepare.py index 7af7645..2c26495 100644 --- a/pydiploy/prepare.py +++ b/pydiploy/prepare.py @@ -28,11 +28,19 @@ def tag(version): """ Defines tag to deploy """ if "pydiploy_version" in env: - fabric.api.abort(fabric.colors.red("tag should be set before calling goal (ex: fab tag:master test deploy)")) + fabric.api.abort( + fabric.colors.red( + "tag should be set before calling goal (ex: fab tag:master test deploy)" + ) + ) if check_tag_exist(version): env.tag = version else: - fabric.api.abort(fabric.colors.red("tag/branch provided is not in the repository please fix this first")) + fabric.api.abort( + fabric.colors.red( + "tag/branch provided is not in the repository please fix this first" + ) + ) def init_params(application_type='default'): @@ -44,7 +52,9 @@ def init_params(application_type='default'): PARAMS[application_type]['optional_params'], ) except KeyError: - fabric.api.abort(fabric.colors.red("application_type '%s' doesn't exists" % application_type)) + fabric.api.abort( + fabric.colors.red("application_type '%s' doesn't exists" % application_type) + ) return required_params, optional_params @@ -61,7 +71,10 @@ def build_env(): \nBe sure that your fabfile complies last pydiploy evolutions. \ \nContinue at your own risks ! \ \n\nDo you want to continue?" - % (fabric.colors.red(env.req_pydiploy_version), fabric.colors.red(__version__)), + % ( + fabric.colors.red(env.req_pydiploy_version), + fabric.colors.red(__version__), + ), default=False, ): fabric.api.abort("Aborting at user request.") @@ -80,13 +93,19 @@ def build_env(): env.remote_project_dir = os.path.join(env.remote_home, env.application_name) if "tag" in env: - env.local_tmp_root_app = os.path.join(env.local_tmp_dir, '%(application_name)s-%(tag)s' % env) - env.local_tmp_root_app_package = os.path.join(env.local_tmp_root_app, env.root_package_name) + env.local_tmp_root_app = os.path.join( + env.local_tmp_dir, '%(application_name)s-%(tag)s' % env + ) + env.local_tmp_root_app_package = os.path.join( + env.local_tmp_root_app, env.root_package_name + ) env.remote_current_path = os.path.join(env.remote_project_dir, 'current') env.remote_releases_path = os.path.join(env.remote_project_dir, 'releases') env.remote_shared_path = os.path.join(env.remote_project_dir, 'shared') - env.remote_base_package_dir = os.path.join(env.remote_current_path, env.root_package_name) + env.remote_base_package_dir = os.path.join( + env.remote_current_path, env.root_package_name + ) env.remote_settings_dir = os.path.join(env.remote_base_package_dir, 'settings') env.remote_settings_file = os.path.join(env.remote_settings_dir, '%s.py' % env.goal) env.lib_path = os.path.dirname(__file__) @@ -118,7 +137,9 @@ def build_env(): if not test_config(verbose=verbose_value, application_type=application_type): if not fabric.contrib.console.confirm( - "Configuration test %s! Do you want to continue?" % fabric.colors.red('failed'), default=False + "Configuration test %s! Do you want to continue?" + % fabric.colors.red('failed'), + default=False, ): fabric.api.abort("Aborting at user request.") @@ -147,9 +168,15 @@ def test_config(verbose=True, application_type='default'): ) for param, desc in sorted(req_params.items()): if param not in env: - err.append("%s -> %s : missing" % (param.ljust(max_req_param_length), desc.ljust(max_req_desc_length))) + err.append( + "%s -> %s : missing" + % (param.ljust(max_req_param_length), desc.ljust(max_req_desc_length)) + ) elif not bool(env[param]): - err.append("%s -> %s : not set" % (param.ljust(max_req_param_length), desc.ljust(max_req_desc_length))) + err.append( + "%s -> %s : not set" + % (param.ljust(max_req_param_length), desc.ljust(max_req_desc_length)) + ) elif verbose: req_parameters.append((param, env[param], desc)) @@ -162,10 +189,13 @@ def test_config(verbose=True, application_type='default'): if err: err_nb = len(err) if err_nb == len(req_params): - fabric.api.puts('You need to configure correctly the fabfile please RTFM first !') + fabric.api.puts( + 'You need to configure correctly the fabfile please RTFM first !' + ) else: fabric.api.puts( - 'Config test failed (%s error%s) for role %s:' % (err_nb, 's' if err_nb > 1 else '', current_role) + 'Config test failed (%s error%s) for role %s:' + % (err_nb, 's' if err_nb > 1 else '', current_role) ) fabric.api.puts('%s\n\n* %s\n' % ('-' * 30, '\n* '.join(err))) fabric.api.puts('Please fix them or continue with possible errors.') @@ -173,7 +203,10 @@ def test_config(verbose=True, application_type='default'): elif verbose: fabric.api.puts('\n\nRequired parameters list : \n\n') for param, value, description in req_parameters: - fabric.api.puts('* %s %s' % (param.ljust(max_req_param_length), fabric.colors.green(value))) + fabric.api.puts( + '* %s %s' + % (param.ljust(max_req_param_length), fabric.colors.green(value)) + ) fabric.api.puts('\n\nOptional parameters list : \n\n') if len(opt_parameters): for param, value, description in opt_parameters: @@ -186,7 +219,8 @@ def test_config(verbose=True, application_type='default'): else: fabric.api.puts("No optional parameter found") fabric.api.puts( - '\n\nRole : %s -> configuration %s!\n\n' % (fabric.colors.green(current_role), fabric.colors.green("OK")) + '\n\nRole : %s -> configuration %s!\n\n' + % (fabric.colors.green(current_role), fabric.colors.green("OK")) ) return True @@ -208,7 +242,9 @@ def check_req_pydiploy_version(): if "req_pydiploy_version" in env: major_version_installed = __version_info__[0:2] - major_version_required = tuple([int(num) for num in env.req_pydiploy_version.split('.')]) + major_version_required = tuple( + [int(num) for num in env.req_pydiploy_version.split('.')] + ) if major_version_installed != major_version_required[0:2]: return False return True @@ -243,7 +279,9 @@ def process_releases(): """ Populates env vars for releases (current, old...) """ if fabtools.files.is_dir(env.remote_releases_path): env.releases = sorted( - fabric.api.run('ls -x %(releases_path)s' % {'releases_path': env.remote_releases_path}).split() + fabric.api.run( + 'ls -x %(releases_path)s' % {'releases_path': env.remote_releases_path} + ).split() ) if len(env.releases) >= 1: env.current_revision = env.releases[-1] @@ -252,9 +290,15 @@ def process_releases(): 'current_revision': env.current_revision, } # warning previous settings file before deployement !!!!! - env.previous_release_base_package_dir = os.path.join(env.current_release, env.root_package_name) - env.previous_release_settings_dir = os.path.join(env.previous_release_base_package_dir, 'settings') - env.previous_settings_file = os.path.join(env.previous_release_settings_dir, '%s.py' % env.goal) + env.previous_release_base_package_dir = os.path.join( + env.current_release, env.root_package_name + ) + env.previous_release_settings_dir = os.path.join( + env.previous_release_base_package_dir, 'settings' + ) + env.previous_settings_file = os.path.join( + env.previous_release_settings_dir, '%s.py' % env.goal + ) fabric.api.puts("settings :", env.previous_settings_file) if len(env.releases) > 1: diff --git a/pydiploy/require/apache.py b/pydiploy/require/apache.py index 33101e7..d901bcc 100644 --- a/pydiploy/require/apache.py +++ b/pydiploy/require/apache.py @@ -19,8 +19,9 @@ def root_web(): """ Creates web root for webserver """ - fabtools.require.files.directory(env.remote_static_root, use_sudo=True, - owner='root', group='root', mode='755') + fabtools.require.files.directory( + env.remote_static_root, use_sudo=True, owner='root', group='root', mode='755' + ) @do_verbose @@ -75,9 +76,10 @@ def apache_confirm_start(): current_role = pydiploy.prepare._get_current_role() if fabric.contrib.console.confirm( "\napache on %s (role %s) seems not to be started ! \ - \n\nDo you want to try to start it?" % - (fabric.colors.red(env.host), - fabric.colors.red(current_role)), default=False): + \n\nDo you want to try to start it?" + % (fabric.colors.red(env.host), fabric.colors.red(current_role)), + default=False, + ): fabtools.service.start('apache2') @@ -99,16 +101,17 @@ def up_site_config(): apache_enabled = os.path.join(apache_root, 'sites-enabled') app_conf = os.path.join(apache_available, '%s.conf' % env.server_name) - fabtools.files.upload_template('apache_host.conf.tpl', - app_conf, - context=env, - template_dir=os.path.join( - env.lib_path, 'templates'), - use_jinja=True, - use_sudo=True, - user='root', - chown=True, - mode='644') + fabtools.files.upload_template( + 'apache_host.conf.tpl', + app_conf, + context=env, + template_dir=os.path.join(env.lib_path, 'templates'), + use_jinja=True, + use_sudo=True, + user='root', + chown=True, + mode='644', + ) fabric.api.execute(apache_enable_site, env.server_name) @@ -121,16 +124,17 @@ def down_site_config(): apache_available = os.path.join(apache_root, 'sites-available') app_conf = os.path.join(apache_available, '%s_down.conf' % env.server_name) - fabtools.files.upload_template('apache_host_down.conf.tpl', - app_conf, - context=env, - template_dir=os.path.join( - env.lib_path, 'templates'), - use_jinja=True, - use_sudo=True, - user='root', - chown=True, - mode='644') + fabtools.files.upload_template( + 'apache_host_down.conf.tpl', + app_conf, + context=env, + template_dir=os.path.join(env.lib_path, 'templates'), + use_jinja=True, + use_sudo=True, + user='root', + chown=True, + mode='644', + ) # fabric.api.execute(upload_maintenance_page) @@ -193,25 +197,26 @@ def set_website_down(): def upload_maintenance_page(): """ Uploads and forges maintenance.html according to template """ - maintenance_file = os.path.join(env.remote_static_root, - env.application_name, - 'maintenance.html') + maintenance_file = os.path.join( + env.remote_static_root, env.application_name, 'maintenance.html' + ) vars_required = ['maintenance_text', 'maintenance_title'] for v in vars_required: if v in env: env[v] = env[v].decode('utf-8') - fabtools.files.upload_template('maintenance.html.tpl', - maintenance_file, - context=env, - template_dir=os.path.join( - env.lib_path, 'templates'), - use_jinja=True, - use_sudo=True, - user='root', - chown=True, - mode='644') + fabtools.files.upload_template( + 'maintenance.html.tpl', + maintenance_file, + context=env, + template_dir=os.path.join(env.lib_path, 'templates'), + use_jinja=True, + use_sudo=True, + user='root', + chown=True, + mode='644', + ) @do_verbose @@ -238,7 +243,8 @@ def apache_enable_site(site=None, restart=False): if site: fabtools.require.apache.site_enabled( - site) if restart else fabtools.require.apache.site_enabled(site) + site + ) if restart else fabtools.require.apache.site_enabled(site) @do_verbose diff --git a/pydiploy/require/bottle/command.py b/pydiploy/require/bottle/command.py index 28722c9..ee5b848 100644 --- a/pydiploy/require/bottle/command.py +++ b/pydiploy/require/bottle/command.py @@ -26,7 +26,8 @@ def bottle_prepare(): with fabtools.python.virtualenv(env.remote_virtualenv_dir): with fabric.api.cd(env.remote_current_path): with fabric.api.settings(sudo_user=env.remote_owner): - fabric.api.sudo('cp -R %s/static assets' % env.root_package_name) + fabric.api.sudo('cp -R %s/static assets' % env.root_package_name) - fabric.api.get(os.path.join(env.remote_current_path, 'assets'), - local_path=env.local_tmp_dir) + fabric.api.get( + os.path.join(env.remote_current_path, 'assets'), local_path=env.local_tmp_dir + ) diff --git a/pydiploy/require/bottle/utils.py b/pydiploy/require/bottle/utils.py index b3e6485..76754d5 100644 --- a/pydiploy/require/bottle/utils.py +++ b/pydiploy/require/bottle/utils.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import os import re + import fabric import fabtools from fabric.api import env @@ -43,13 +44,12 @@ def extract_settings(): def app_settings(**kwargs): """ Manages django settings file """ - settings_present = fabtools.files.is_file(path=env.previous_settings_file, - use_sudo=True) + settings_present = fabtools.files.is_file( + path=env.previous_settings_file, use_sudo=True + ) # if values are set within the --set option on command line - kwargs.update({ - key: value for key, value in env.items() if key in env.map_settings - }) + kwargs.update({key: value for key, value in env.items() if key in env.map_settings}) if settings_present: fabric.api.execute(extract_settings) @@ -65,45 +65,48 @@ def app_settings(**kwargs): fabric.api.require(*env.map_settings.keys()) settings_dir = os.path.join(env.local_tmp_root_app_package, 'settings') - fabtools.files.upload_template('%s.py' % env.goal, - env.remote_settings_file, - template_dir=settings_dir, - context=env, - use_sudo=True, - user=env.remote_owner, - mode='644', - chown=True, - use_jinja=True) + fabtools.files.upload_template( + '%s.py' % env.goal, + env.remote_settings_file, + template_dir=settings_dir, + context=env, + use_sudo=True, + user=env.remote_owner, + mode='644', + chown=True, + use_jinja=True, + ) @do_verbose def deploy_environ_file(): """ Uploads environ.py template on remote """ - fabtools.files.upload_template('environ.py', - os.path.join( - env.remote_base_package_dir, - 'environ.py'), - template_dir=env.local_tmp_root_app_package, - context=env, - use_sudo=True, - user=env.remote_owner, - chown=True, - mode='644', - use_jinja=True) + fabtools.files.upload_template( + 'environ.py', + os.path.join(env.remote_base_package_dir, 'environ.py'), + template_dir=env.local_tmp_root_app_package, + context=env, + use_sudo=True, + user=env.remote_owner, + chown=True, + mode='644', + use_jinja=True, + ) @do_verbose def deploy_wsgi_file(): """ Uploads wsgi.py template on remote """ - fabtools.files.upload_template('wsgi.py', - os.path.join( - env.remote_base_package_dir, 'wsgi.py'), - template_dir=env.local_tmp_root_app_package, - context=env, - use_sudo=True, - user=env.remote_owner, - chown=True, - mode='644', - use_jinja=True) + fabtools.files.upload_template( + 'wsgi.py', + os.path.join(env.remote_base_package_dir, 'wsgi.py'), + template_dir=env.local_tmp_root_app_package, + context=env, + use_sudo=True, + user=env.remote_owner, + chown=True, + mode='644', + use_jinja=True, + ) diff --git a/pydiploy/require/circus.py b/pydiploy/require/circus.py index b8f7b97..8fcea7c 100644 --- a/pydiploy/require/circus.py +++ b/pydiploy/require/circus.py @@ -34,13 +34,21 @@ def circus_pkg(update=False): # fabtools.require.deb.ppa('ppa:chris-lea/zeromq') # fabtools.require.deb.ppa('ppa:chris-lea/libpgm') - if fabtools.system.distrib_id() == 'Ubuntu' and fabtools.system.distrib_release() >= '18.04': + if ( + fabtools.system.distrib_id() == 'Ubuntu' + and fabtools.system.distrib_release() >= '18.04' + ): fabtools.require.deb.packages(['libzmq3-dev', 'libevent-dev'], update=update) - if fabtools.system.distrib_id() == 'Ubuntu' and fabtools.system.distrib_release() < '18.04': + if ( + fabtools.system.distrib_id() == 'Ubuntu' + and fabtools.system.distrib_release() < '18.04' + ): fabtools.require.deb.packages(['libzmq-dev', 'libevent-dev'], update=update) - fabtools.require.python.install(env.get('circus_package_name', 'circus'), use_sudo=True, upgrade=update) + fabtools.require.python.install( + env.get('circus_package_name', 'circus'), use_sudo=True, upgrade=update + ) if 'no_circus_web' not in env or not env.no_circus_web: fabtools.require.python.install('circus-web', use_sudo=True, upgrade=update) @@ -48,7 +56,9 @@ def circus_pkg(update=False): # install circus backend sets in fabfile if 'circus_backend' in env: - fabtools.require.python.install(env.circus_backend, use_sudo=True, upgrade=update) + fabtools.require.python.install( + env.circus_backend, use_sudo=True, upgrade=update + ) # base configuration file for circus fabtools.files.upload_template( @@ -150,7 +160,10 @@ def app_reload(): with fabric.api.settings(sudo_user=env.remote_owner): fabric.api.sudo('circusctl reloadconfig') app_installed = fabtools.files.is_file( - path=os.path.join(env.remote_home, '.circus.d', '%s.ini' % env.application_name), use_sudo=True + path=os.path.join( + env.remote_home, '.circus.d', '%s.ini' % env.application_name + ), + use_sudo=True, ) with warn_only(), hide(): fabric.api.sudo('circusctl restart %s' % env.application_name) diff --git a/pydiploy/require/databases/ldap.py b/pydiploy/require/databases/ldap.py index c3aacc2..7fcb32d 100644 --- a/pydiploy/require/databases/ldap.py +++ b/pydiploy/require/databases/ldap.py @@ -20,5 +20,5 @@ def ldap_pkg(use_sudo=False, user=None): fabtools.require.deb.package('libsasl2-dev', update=True) fabtools.require.deb.package('libssl-dev', update=True) fabtools.require.python.package( - 'python-ldap', upgrade=True, use_sudo=use_sudo, - user=user) + 'python-ldap', upgrade=True, use_sudo=use_sudo, user=user + ) diff --git a/pydiploy/require/databases/mongodb.py b/pydiploy/require/databases/mongodb.py index dace7ff..4f5f18f 100644 --- a/pydiploy/require/databases/mongodb.py +++ b/pydiploy/require/databases/mongodb.py @@ -17,10 +17,15 @@ @do_verbose def install_mongodb(): if not pydiploy.require.system.package_installed('mongodb-10gen'): - fabtools.require.deb.source('mongodb', - 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', - 'dist', '10gen') + fabtools.require.deb.source( + 'mongodb', + 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', + 'dist', + '10gen', + ) - fabric.api.sudo('apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10') + fabric.api.sudo( + 'apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10' + ) fabtools.require.deb.uptodate_index(quiet=True) fabtools.require.deb.package('mongodb-10gen') diff --git a/pydiploy/require/databases/mysql.py b/pydiploy/require/databases/mysql.py index 19bdc77..932a176 100644 --- a/pydiploy/require/databases/mysql.py +++ b/pydiploy/require/databases/mysql.py @@ -29,14 +29,20 @@ def install_mysql_server(version=None, password=None): def add_mysql_user(name, password, host='localhost', **kwargs): """ Adds mysql user """ if not fabtools.mysql.user_exists(name): - fabtools.mysql.create_user( - name, password=password, host=host, **kwargs) + fabtools.mysql.create_user(name, password=password, host=host, **kwargs) @do_verbose -def add_mysql_database(name, owner=None, owner_host='localhost', - charset='utf8', collate='utf8_general_ci', **kwargs): +def add_mysql_database( + name, + owner=None, + owner_host='localhost', + charset='utf8', + collate='utf8_general_ci', + **kwargs +): """ Adds mysql database """ if not fabtools.mysql.database_exists(name): - fabtools.mysql.create_database(name, owner=owner, charset=charset, - collate=collate, **kwargs) + fabtools.mysql.create_database( + name, owner=owner, charset=charset, collate=collate, **kwargs + ) diff --git a/pydiploy/require/databases/oracle.py b/pydiploy/require/databases/oracle.py index fdf0843..b25cc3c 100644 --- a/pydiploy/require/databases/oracle.py +++ b/pydiploy/require/databases/oracle.py @@ -30,10 +30,17 @@ def install_oracle_client(): * env.oracle_packages : name(s) of zip file(s) for oracle's packages to deploy """ - if all([e in env.keys() for e in ('oracle_client_version', - 'oracle_download_url', - 'oracle_remote_dir', - 'oracle_packages')]): + if all( + [ + e in env.keys() + for e in ( + 'oracle_client_version', + 'oracle_download_url', + 'oracle_remote_dir', + 'oracle_packages', + ) + ] + ): # system libs and goodies installation fabtools.require.deb.packages(['libaio-dev', 'unzip']) @@ -43,20 +50,19 @@ def install_oracle_client(): use_sudo=True, owner=env.remote_owner, group=env.remote_group, - mode='750') + mode='750', + ) # get oracle's zip file(s) and unzip with fabric.api.cd(env.remote_home): for package in env.oracle_packages: - fabric.api.sudo('wget -c %s%s' % - (env.oracle_download_url, package)) - fabric.api.sudo('unzip %s -d %s' % - (package, env.oracle_remote_dir)) - fabric.api.sudo( - 'rm %s' % os.path.join(env.remote_home, package)) + fabric.api.sudo('wget -c %s%s' % (env.oracle_download_url, package)) + fabric.api.sudo('unzip %s -d %s' % (package, env.oracle_remote_dir)) + fabric.api.sudo('rm %s' % os.path.join(env.remote_home, package)) oracle_dir = 'instantclient_%s' % '_'.join( - env.oracle_client_version.split('.')[:2]) + env.oracle_client_version.split('.')[:2] + ) oracle_root_path = os.path.join(env.oracle_remote_dir, oracle_dir) oracle_full_path = os.path.join(env.remote_home, oracle_root_path) @@ -72,11 +78,14 @@ def install_oracle_client(): ) fabric.api.sudo('pwd') - fabric.api.sudo('echo \'%s\' >> .bashrc' % - oracle_conf.substitute(oracle_dir=oracle_full_path)) + fabric.api.sudo( + 'echo \'%s\' >> .bashrc' + % oracle_conf.substitute(oracle_dir=oracle_full_path) + ) fabric.api.sudo('source .bashrc') fabric.api.sudo( - 'echo %s > /etc/ld.so.conf.d/oracle.conf' % oracle_full_path) + 'echo %s > /etc/ld.so.conf.d/oracle.conf' % oracle_full_path + ) fabric.api.sudo('ldconfig') else: fabric.api.abort('Please provide parameters for oracle installation !') diff --git a/pydiploy/require/databases/postgres.py b/pydiploy/require/databases/postgres.py index bc7d640..c3ed957 100644 --- a/pydiploy/require/databases/postgres.py +++ b/pydiploy/require/databases/postgres.py @@ -31,19 +31,35 @@ def install_postgres_server(update=False): @do_verbose -def add_postgres_user(name, password, superuser=False, createdb=False, - createrole=False, inherit=True, login=True, - connection_limit=None, encrypted_password=False, - verbose=True): +def add_postgres_user( + name, + password, + superuser=False, + createdb=False, + createrole=False, + inherit=True, + login=True, + connection_limit=None, + encrypted_password=False, + verbose=True, +): """ Installs postgres server on remote """ if "verbose_output" in env: verbose = env.verbose_output msg = '' if not fabtools.postgres.user_exists(name): - fabtools.require.postgres.user(name, password, superuser, createdb, - createrole, inherit, login, - connection_limit, encrypted_password) + fabtools.require.postgres.user( + name, + password, + superuser, + createdb, + createrole, + inherit, + login, + connection_limit, + encrypted_password, + ) msg = 'Add %s user for postgresql !' % fabric.colors.green(name) else: msg = 'User %s already exists !' % fabric.colors.red(name) @@ -53,21 +69,30 @@ def add_postgres_user(name, password, superuser=False, createdb=False, @do_verbose -def add_postgres_database(name, owner, template='template0', encoding='UTF8', - locale='fr_FR.UTF-8', verbose=True): +def add_postgres_database( + name, + owner, + template='template0', + encoding='UTF8', + locale='fr_FR.UTF-8', + verbose=True, +): """ Adds postgresql database on remote """ if "verbose_output" in env: verbose = env.verbose_output msg = '' if not fabtools.postgres.database_exists(name): - fabtools.require.postgres.database(name, owner, template, encoding, - locale) + fabtools.require.postgres.database(name, owner, template, encoding, locale) msg = 'Add %s database for owner %s !' % ( - fabric.colors.green(name), fabric.colors.green(owner)) + fabric.colors.green(name), + fabric.colors.green(owner), + ) else: msg = 'Database %s for owner %s already exists !' % ( - fabric.colors.red(name), fabric.colors.red(owner)) + fabric.colors.red(name), + fabric.colors.red(owner), + ) if verbose: fabric.api.puts(msg) diff --git a/pydiploy/require/databases/sap.py b/pydiploy/require/databases/sap.py index 6d88db2..4c1de68 100644 --- a/pydiploy/require/databases/sap.py +++ b/pydiploy/require/databases/sap.py @@ -29,29 +29,27 @@ def install_sap_client(): """ sap_lib_path = join(sep, 'usr', 'sap') - if all([e in env.keys() for e in ('sap_download_url', - 'sap_packages')]): + if all([e in env.keys() for e in ('sap_download_url', 'sap_packages')]): # system libs and goodies installation fabtools.require.deb.packages(['libstdc++5']) - fabtools.require.files.directory(path=sap_lib_path, - use_sudo=True, - mode='755') + fabtools.require.files.directory(path=sap_lib_path, use_sudo=True, mode='755') # get oracle's zip file(s) and unzip with fabric.api.cd(sap_lib_path): for package in env.sap_packages: - fabric.api.sudo('wget -c %s%s' % - (env.sap_download_url, package)) + fabric.api.sudo('wget -c %s%s' % (env.sap_download_url, package)) fabric.api.sudo('tar xvf %s' % package) fabric.api.sudo('chmod -R 755 rfcsdk') fabric.api.sudo('rm %s' % package) with fabric.api.cd(join(sep, 'lib')): if not fabtools.files.is_link('librfccm.so', use_sudo=True): - fabric.api.sudo('ln -s %s .' % join(sap_lib_path, 'rfcsdk', - 'lib', 'librfccm.so')) + fabric.api.sudo( + 'ln -s %s .' + % join(sap_lib_path, 'rfcsdk', 'lib', 'librfccm.so') + ) else: fabric.api.abort('Please provide parameters for sap installation !') diff --git a/pydiploy/require/databases/sqlite.py b/pydiploy/require/databases/sqlite.py index 1b8c2c3..3360337 100644 --- a/pydiploy/require/databases/sqlite.py +++ b/pydiploy/require/databases/sqlite.py @@ -18,5 +18,5 @@ def sqlite3_pkg(use_sudo=False, user=None): fabtools.require.deb.package('libsqlite3-dev', update=True) fabtools.require.python.package( - 'pysqlite', upgrade=True, use_sudo=use_sudo, - user=user) + 'pysqlite', upgrade=True, use_sudo=use_sudo, user=user + ) diff --git a/pydiploy/require/django/command.py b/pydiploy/require/django/command.py index 65b1a76..8c9f018 100644 --- a/pydiploy/require/django/command.py +++ b/pydiploy/require/django/command.py @@ -6,12 +6,13 @@ import datetime import os +from distutils.version import LooseVersion import fabric import fabtools from fabric.api import env, hide + from pydiploy.decorators import do_verbose -from distutils.version import LooseVersion @do_verbose @@ -44,8 +45,9 @@ def django_prepare(): # ' -i '.join(ignore)) fabric.api.sudo('python manage.py collectstatic --noinput') - fabric.api.get(os.path.join(env.remote_current_path, 'assets'), - local_path=env.local_tmp_dir) + fabric.api.get( + os.path.join(env.remote_current_path, 'assets'), local_path=env.local_tmp_dir + ) @do_verbose @@ -62,9 +64,11 @@ def django_dump_database(): with fabric.api.cd(env.remote_current_path): with fabric.api.settings(sudo_user=env.remote_owner): dump_name = '%s.json' % datetime.datetime.today().strftime( - "%Y_%m_%d-%H%M") + "%Y_%m_%d-%H%M" + ) fabric.api.sudo( - 'python manage.py dumpdata --indent=4 > /tmp/%s ' % dump_name) + 'python manage.py dumpdata --indent=4 > /tmp/%s ' % dump_name + ) fabric.api.get('/tmp/%s' % dump_name, local_path=env.dest_path) @@ -84,11 +88,12 @@ def django_get_version(): # hopefully never compare with django_version=0 :) django_version = 0 - + with fabtools.python.virtualenv(env.remote_virtualenv_dir): with fabric.api.cd(env.remote_current_path): with fabric.api.settings(sudo_user=env.remote_owner): django_version = fabric.api.sudo( - 'python -c "import django;print(django.get_version())"') + 'python -c "import django;print(django.get_version())"' + ) return django_version diff --git a/pydiploy/require/django/utils.py b/pydiploy/require/django/utils.py index 7f3f036..9f298c9 100644 --- a/pydiploy/require/django/utils.py +++ b/pydiploy/require/django/utils.py @@ -5,9 +5,10 @@ import os import random import re + import fabric import fabtools -from fabric.api import env +from fabric.api import env, hide, warn_only from pydiploy.decorators import do_verbose @@ -16,8 +17,7 @@ def generate_secret_key(): """ Generates the django's secret key. """ letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*(-_=+)' - random_letters = map(lambda i: random.SystemRandom().choice(letters), - range(50)) + random_letters = map(lambda i: random.SystemRandom().choice(letters), range(50)) env.secret_key = ''.join(random_letters) @@ -27,9 +27,11 @@ def extract_settings(): """ Extracts settings from django settings files. """ # get the remote file - fabric.api.get(env.previous_settings_file, local_path=env.local_tmp_dir) - settings_file = os.path.join(env.local_tmp_dir, '%s.py' % env.goal) + # with warn_only(), hide(): + with hide('everything'): + fabric.api.get(env.previous_settings_file, local_path=env.local_tmp_dir) + settings_file = os.path.join(env.local_tmp_dir, '%s.py' % env.goal) # open and read the data from the downloaded file with open(settings_file, 'r') as settings_fh: settings_data = settings_fh.readlines() @@ -58,13 +60,10 @@ def extract_settings(): def app_settings(**kwargs): """ Manages django settings file """ - settings_present = fabtools.files.is_file(path=env.previous_settings_file, - use_sudo=True) + settings_present = fabtools.files.is_file(path=env.previous_settings_file, use_sudo=True) # if values are set within the --set option on command line - kwargs.update({ - key: value for key, value in env.items() if key in env.map_settings - }) + kwargs.update({key: value for key, value in env.items() if key in env.map_settings}) if settings_present: fabric.api.execute(extract_settings) @@ -80,44 +79,48 @@ def app_settings(**kwargs): fabric.api.require(*env.map_settings.keys()) settings_dir = os.path.join(env.local_tmp_root_app_package, 'settings') - fabtools.files.upload_template('%s.py' % env.goal, - env.remote_settings_file, - template_dir=settings_dir, - context=env, - use_sudo=True, - user=env.remote_owner, - mode='644', - chown=True, - use_jinja=True) + fabtools.files.upload_template( + '%s.py' % env.goal, + env.remote_settings_file, + template_dir=settings_dir, + context=env, + use_sudo=True, + user=env.remote_owner, + mode='644', + chown=True, + use_jinja=True, + ) @do_verbose def deploy_manage_file(): """ uploads manage.py template on remote """ - fabtools.files.upload_template('manage.py', - os.path.join( - env.remote_current_release, 'manage.py'), - template_dir=env.local_tmp_root_app, - context=env, - use_sudo=True, - user=env.remote_owner, - chown=True, - mode='744', - use_jinja=True) + fabtools.files.upload_template( + 'manage.py', + os.path.join(env.remote_current_release, 'manage.py'), + template_dir=env.local_tmp_root_app, + context=env, + use_sudo=True, + user=env.remote_owner, + chown=True, + mode='744', + use_jinja=True, + ) @do_verbose def deploy_wsgi_file(): """ Uploads wsgi.py template on remote """ - fabtools.files.upload_template('wsgi.py', - os.path.join( - env.remote_base_package_dir, 'wsgi.py'), - template_dir=env.local_tmp_root_app_package, - context=env, - use_sudo=True, - user=env.remote_owner, - chown=True, - mode='644', - use_jinja=True) + fabtools.files.upload_template( + 'wsgi.py', + os.path.join(env.remote_base_package_dir, 'wsgi.py'), + template_dir=env.local_tmp_root_app_package, + context=env, + use_sudo=True, + user=env.remote_owner, + chown=True, + mode='644', + use_jinja=True, + ) diff --git a/pydiploy/require/git.py b/pydiploy/require/git.py index 631d0e7..edf57ab 100644 --- a/pydiploy/require/git.py +++ b/pydiploy/require/git.py @@ -16,7 +16,14 @@ @do_verbose def archive( - filename, path='/tmp', format="tar.gz", tag="HEAD", remote="", prefix="", project_path=".", specific_folder="" + filename, + path='/tmp', + format="tar.gz", + tag="HEAD", + remote="", + prefix="", + project_path=".", + specific_folder="", ): """ Creates an archive from a git repository or directly from a project @@ -120,6 +127,10 @@ def collect_branches(project_path='.', remote=""): def check_tag_exist(tag=None): """ Checks if a tag/branch exists in the repository """ if tag: - if tag not in collect_branches() and tag not in collect_tags() and 'no_tag_check' not in env: + if ( + tag not in collect_branches() + and tag not in collect_tags() + and 'no_tag_check' not in env + ): return False return True diff --git a/pydiploy/require/nginx.py b/pydiploy/require/nginx.py index eb9d8d4..8f5b1fb 100644 --- a/pydiploy/require/nginx.py +++ b/pydiploy/require/nginx.py @@ -9,6 +9,7 @@ import pydiploy from fabric.api import env, warn_only from pydiploy.decorators import do_verbose + from .system import is_systemd @@ -16,8 +17,9 @@ def root_web(): """ Creates web root for webserver """ - fabtools.require.files.directory(env.remote_static_root, use_sudo=True, - owner='root', group='root', mode='755') + fabtools.require.files.directory( + env.remote_static_root, use_sudo=True, owner='root', group='root', mode='755' + ) @do_verbose @@ -31,7 +33,9 @@ def nginx_pkg(update=False): def nginx_start(): """ Starts nginx """ - if not nginx_started() and ('nginx_force_start' not in env or not env.nginx_force_start): + if not nginx_started() and ( + 'nginx_force_start' not in env or not env.nginx_force_start + ): fabric.api.puts("Nginx is not started") else: if is_systemd(): @@ -40,7 +44,6 @@ def nginx_start(): fabtools.service.start('nginx') - @do_verbose def nginx_reload(): """ Starts/reloads nginx """ @@ -79,7 +82,9 @@ def nginx_started(): if is_systemd(): # return fabtools.systemd.is_running('nginx') with warn_only(): - return 'inactive' not in fabric.api.sudo('systemctl is-active nginx.service') + return 'inactive' not in fabric.api.sudo( + 'systemctl is-active nginx.service' + ) else: return fabtools.service.is_running('nginx') @@ -90,9 +95,11 @@ def web_static_files(): fabric.contrib.project.rsync_project( os.path.join(env.remote_static_root, env.application_name), - os.path.join(env.local_tmp_dir, 'assets/'), delete=True, + os.path.join(env.local_tmp_dir, 'assets/'), + delete=True, extra_opts='--rsync-path="sudo rsync" --exclude="maintenance.html"', - ssh_opts='-t') + ssh_opts='-t', + ) @do_verbose @@ -120,19 +127,19 @@ def up_site_config(): nginx_enabled = os.path.join(nginx_root, 'sites-enabled') app_conf = os.path.join(nginx_available, '%s.conf' % env.server_name) - fabtools.files.upload_template('nginx.conf.tpl', - app_conf, - context=env, - template_dir=os.path.join( - env.lib_path, 'templates'), - use_jinja=True, - use_sudo=True, - user='root', - chown=True, - mode='644') - - if not fabtools.files.is_link('%s/%s.conf' % (nginx_enabled, - env.server_name)): + fabtools.files.upload_template( + 'nginx.conf.tpl', + app_conf, + context=env, + template_dir=os.path.join(env.lib_path, 'templates'), + use_jinja=True, + use_sudo=True, + user='root', + chown=True, + mode='644', + ) + + if not fabtools.files.is_link('%s/%s.conf' % (nginx_enabled, env.server_name)): with fabric.api.cd(nginx_enabled): fabric.api.sudo('ln -s %s .' % app_conf) @@ -145,16 +152,17 @@ def down_site_config(): nginx_available = os.path.join(nginx_root, 'sites-available') app_conf = os.path.join(nginx_available, '%s_down.conf' % env.server_name) - fabtools.files.upload_template('nginx_down.conf.tpl', - app_conf, - context=env, - template_dir=os.path.join( - env.lib_path, 'templates'), - use_jinja=True, - use_sudo=True, - user='root', - chown=True, - mode='644') + fabtools.files.upload_template( + 'nginx_down.conf.tpl', + app_conf, + context=env, + template_dir=os.path.join(env.lib_path, 'templates'), + use_jinja=True, + use_sudo=True, + user='root', + chown=True, + mode='644', + ) fabric.api.execute(upload_maintenance_page) @@ -171,13 +179,11 @@ def set_website_up(): if not fabtools.files.is_file(app_conf): fabric.api.execute(up_site_config) - if fabtools.files.is_link('%s/%s_down.conf' % (nginx_enabled, - env.server_name)): + if fabtools.files.is_link('%s/%s_down.conf' % (nginx_enabled, env.server_name)): with fabric.api.cd(nginx_enabled): fabric.api.sudo('rm -f %s_down.conf' % env.server_name) - if not fabtools.files.is_link('%s/%s.conf' % (nginx_enabled, - env.server_name)): + if not fabtools.files.is_link('%s/%s.conf' % (nginx_enabled, env.server_name)): with fabric.api.cd(nginx_enabled): fabric.api.sudo('ln -s %s .' % app_conf) @@ -191,19 +197,16 @@ def set_website_down(): nginx_root = '/etc/nginx' nginx_available = os.path.join(nginx_root, 'sites-available') nginx_enabled = os.path.join(nginx_root, 'sites-enabled') - app_down_conf = os.path.join( - nginx_available, '%s_down.conf' % env.server_name) + app_down_conf = os.path.join(nginx_available, '%s_down.conf' % env.server_name) if not fabtools.files.is_file(app_down_conf): fabric.api.execute(down_site_config) - if fabtools.files.is_link('%s/%s.conf' % (nginx_enabled, - env.server_name)): + if fabtools.files.is_link('%s/%s.conf' % (nginx_enabled, env.server_name)): with fabric.api.cd(nginx_enabled): fabric.api.sudo('rm -f %s.conf' % env.server_name) - if not fabtools.files.is_link('%s/%s_down.conf' % (nginx_enabled, - env.server_name)): + if not fabtools.files.is_link('%s/%s_down.conf' % (nginx_enabled, env.server_name)): with fabric.api.cd(nginx_enabled): fabric.api.sudo('ln -s %s .' % app_down_conf) @@ -214,22 +217,24 @@ def set_website_down(): def upload_maintenance_page(): """ Uploads and forges maintenance.html according to template """ - maintenance_file = os.path.join(env.remote_static_root, - env.application_name, - 'maintenance.html') + maintenance_file = os.path.join( + env.remote_static_root, env.application_name, 'maintenance.html' + ) vars_required = ['maintenance_text', 'maintenance_title'] for v in vars_required: if v in env: env[v] = env[v].decode('utf-8') - fabtools.files.upload_template('maintenance.html.tpl', - maintenance_file, - context=env, - template_dir=os.path.join( - env.lib_path, 'templates'), - use_jinja=True, - use_sudo=True, - user='root', - chown=True, - mode='644') + fabtools.files.upload_template( + 'maintenance.html.tpl', + maintenance_file, + context=env, + template_dir=os.path.join(env.lib_path, 'templates'), + use_jinja=True, + use_sudo=True, + user='root', + chown=True, + mode='644', + ) + diff --git a/pydiploy/require/python/utils.py b/pydiploy/require/python/utils.py index 0701866..3c649ea 100644 --- a/pydiploy/require/python/utils.py +++ b/pydiploy/require/python/utils.py @@ -15,13 +15,18 @@ def python_pkg(update=False): """ Installs python packages and pip """ - if fabtools.system.distrib_id() == 'Ubuntu' and float(fabtools.system.distrib_release()) >= 18.04: + if ( + fabtools.system.distrib_id() == 'Ubuntu' + and float(fabtools.system.distrib_release()) >= 18.04 + ): if env.remote_python_version < 3: fabric.api.sudo('add-apt-repository universe') fabtools.require.deb.packages( [ - 'python2-dev' if env.remote_python_version < 3 else 'python%s-dev' % env.remote_python_version, + 'python2-dev' + if env.remote_python_version < 3 + else 'python%s-dev' % env.remote_python_version, 'python3-pip' if not env.remote_python_version < 3 else 'python2', 'build-essential', ], @@ -35,7 +40,9 @@ def python_pkg(update=False): else: fabtools.require.deb.packages( [ - 'python-dev' if env.remote_python_version < 3 else 'python%s-dev' % env.remote_python_version, + 'python-dev' + if env.remote_python_version < 3 + else 'python%s-dev' % env.remote_python_version, 'python-pip', ], update=update, @@ -49,13 +56,19 @@ def application_dependencies(upgrade_pkg, staging=True): with fabtools.python.virtualenv(env.remote_virtualenv_dir): with fabric.api.cd(env.remote_current_path): - requirements_file = os.path.join('requirements', '%s.txt' % env.goal) if staging else 'requirements.txt' + requirements_file = ( + os.path.join('requirements', '%s.txt' % env.goal) + if staging + else 'requirements.txt' + ) # ugly fix for error when pip install fail and error raises while /home/user/.pip not writable # TODO; maybe use app_name_pip_error.log file ? Or use a setting path for log file ? pip_log = '%s/pip_error.log' % env.remote_home pip_cmd = 'pip --log-file %s' % pip_log if 'oracle_client_version' in env: - oracle_dir = 'instantclient_%s' % '_'.join(env.oracle_client_version.split('.')[:2]) + oracle_dir = 'instantclient_%s' % '_'.join( + env.oracle_client_version.split('.')[:2] + ) oracle_root_path = os.path.join(env.oracle_remote_dir, oracle_dir) oracle_full_path = os.path.join(env.remote_home, oracle_root_path) pip_cmd = 'ORACLE_HOME=%s pip' % oracle_full_path @@ -63,7 +76,12 @@ def application_dependencies(upgrade_pkg, staging=True): with shell("HOME=~%s %s" % (env.remote_owner, env.shell)): # upgrade pip to latest version fabtools.require.python.install( - 'pip', upgrade=True, use_sudo=True, user=env.remote_owner, pip_cmd=pip_cmd, quiet=True + 'pip', + upgrade=True, + use_sudo=True, + user=env.remote_owner, + pip_cmd=pip_cmd, + quiet=True, ) fabtools.python.install_requirements( @@ -75,4 +93,8 @@ def application_dependencies(upgrade_pkg, staging=True): quiet=True, ) - fabric.api.sudo('pip install --log-file %s --quiet -e .' % pip_log, user=env.remote_owner, pty=False) + fabric.api.sudo( + 'pip install --log-file %s --quiet -e .' % pip_log, + user=env.remote_owner, + pty=False, + ) diff --git a/pydiploy/require/python/virtualenv.py b/pydiploy/require/python/virtualenv.py index 2f73b08..e8b7fc8 100644 --- a/pydiploy/require/python/virtualenv.py +++ b/pydiploy/require/python/virtualenv.py @@ -10,13 +10,18 @@ @do_verbose def virtualenv(clear=False): """ Creates virtualenv """ - fabtools.require.files.directory(env.remote_virtualenv_dir, - owner=env.remote_owner, - group=env.remote_group, - use_sudo=True) + fabtools.require.files.directory( + env.remote_virtualenv_dir, + owner=env.remote_owner, + group=env.remote_group, + use_sudo=True, + ) python_bin = '/usr/bin/python%s' % env.remote_python_version with shell("HOME=~%s %s" % (env.remote_owner, env.shell)): - fabtools.require.python.virtualenv(env.remote_virtualenv_dir, - user=env.remote_owner, - clear=clear, use_sudo=True, - venv_python=python_bin) + fabtools.require.python.virtualenv( + env.remote_virtualenv_dir, + user=env.remote_owner, + clear=clear, + use_sudo=True, + venv_python=python_bin, + ) diff --git a/pydiploy/require/releases_manager.py b/pydiploy/require/releases_manager.py index 9cb7b6b..db9b6ac 100644 --- a/pydiploy/require/releases_manager.py +++ b/pydiploy/require/releases_manager.py @@ -18,7 +18,10 @@ def set_current(): """ fabric.api.sudo( "ln -nfs %(current_release)s %(current_path)s" - % {'current_release': env.remote_current_release, 'current_path': env.remote_current_path} + % { + 'current_release': env.remote_current_release, + 'current_path': env.remote_current_path, + } ) @@ -28,15 +31,22 @@ def setup(): Configs stuff for deployement """ fabric.api.sudo( - "mkdir -p %(remote_domain_path)s/{releases,shared}" % {'remote_domain_path': env.remote_project_dir} + "mkdir -p %(remote_domain_path)s/{releases,shared}" + % {'remote_domain_path': env.remote_project_dir} + ) + fabric.api.sudo( + "mkdir -p %(remote_shared_path)s/{config,log}" + % {'remote_shared_path': env.remote_shared_path} ) - fabric.api.sudo("mkdir -p %(remote_shared_path)s/{config,log}" % {'remote_shared_path': env.remote_shared_path}) # extra symlinks if present in settings if env.has_key('extra_symlink_dirs'): for extra_symlink_dir in env.extra_symlink_dirs: fabric.api.sudo( "mkdir -p %(remote_shared_path)s/%(shared_dir)s" - % {'remote_shared_path': env.remote_shared_path, 'shared_dir': os.path.basename(extra_symlink_dir)} + % { + 'remote_shared_path': env.remote_shared_path, + 'shared_dir': os.path.basename(extra_symlink_dir), + } ) fabric.api.execute(pydiploy.require.system.permissions) @@ -54,7 +64,8 @@ def cleanup(): del directories[: env.keep_releases] env.directories = ' '.join( [ - "%(releases_path)s/%(release)s" % {'releases_path': env.remote_releases_path, 'release': release} + "%(releases_path)s/%(release)s" + % {'releases_path': env.remote_releases_path, 'release': release} for release in directories ] ) @@ -73,13 +84,18 @@ def deploy_code(): tag_requested = fabric.api.prompt('Please specify target tag used: ') while not pydiploy.require.git.check_tag_exist(tag_requested): tag_requested = fabric.api.prompt( - 'tag %s unknown please specify valid target tag used: ' % fabric.colors.red(tag_requested) + 'tag %s unknown please specify valid target tag used: ' + % fabric.colors.red(tag_requested) ) env.tag = tag_requested - env.local_tmp_root_app = os.path.join(env.local_tmp_dir, '%(application_name)s-%(tag)s' % env) - env.local_tmp_root_app_package = os.path.join(env.local_tmp_root_app, env.root_package_name) + env.local_tmp_root_app = os.path.join( + env.local_tmp_dir, '%(application_name)s-%(tag)s' % env + ) + env.local_tmp_root_app_package = os.path.join( + env.local_tmp_root_app, env.root_package_name + ) fabric.api.require('tag', provided_by=['tag', 'head']) fabric.api.require('remote_project_dir', provided_by=env.goals) @@ -89,7 +105,9 @@ def deploy_code(): tarball = pydiploy.require.git.archive( env.application_name, prefix='%s/' % archive_prefix, - specific_folder=env.remote_repo_specific_folder if "remote_repo_specific_folder" in env else "", + specific_folder=env.remote_repo_specific_folder + if "remote_repo_specific_folder" in env + else "", tag=env.tag, remote=env.remote_repo_url, ) @@ -100,9 +118,20 @@ def deploy_code(): fabric.api.local('rm -rf %s' % archive_prefix) fabric.api.local('tar xvf %s' % os.path.basename(tarball)) # add deployed.json file - with open('%s/%s_info.json' % (env.local_tmp_root_app, env.application_name), 'w') as f: + with open( + '%s/%s_info.json' % (env.local_tmp_root_app, env.application_name), 'w' + ) as f: data = {} - data['info'] = [{'tag': env.tag, 'python_version': env.remote_python_version,}] + data['info'] = [ + { + 'tag': env.tag, + 'python_version': env.remote_python_version, + 'local_user': env.local_user, + 'app_host': env.host_string, + 'socket_port': env.socket_port, + 'repo_url': env.remote_repo_url, + } + ] json.dump(data, f) @@ -125,14 +154,18 @@ def deploy_code(): '.gitignore', '.gitattributes', ] - exclude_files += ['%s/settings/%s.py' % (env.root_package_name, goal) for goal in env.goals] + exclude_files += [ + '%s/settings/%s.py' % (env.root_package_name, goal) for goal in env.goals + ] if env.has_key('excluded_files'): exclude_files += env.excluded_files if env.has_key('cfg_shared_files'): for cfg_shared_file in env.cfg_shared_files: cfg_present = fabtools.files.is_file( - path='%s/config/%s' % (env.remote_shared_path, os.path.basename(cfg_shared_file)), use_sudo=True + path='%s/config/%s' + % (env.remote_shared_path, os.path.basename(cfg_shared_file)), + use_sudo=True, ) if cfg_present is None: fabtools.files.upload_template( @@ -163,11 +196,17 @@ def deploy_code(): # Wrong repository url (git archive is empty) if result.return_code == 23: - fabric.api.abort(fabric.colors.red("Unable to use repository, please check repository url !")) + fabric.api.abort( + fabric.colors.red("Unable to use repository, please check repository url !") + ) fabric.api.sudo( 'chown -R %(user)s:%(group)s %(project_dir)s' - % {'user': env.remote_owner, 'group': env.remote_group, 'project_dir': env.remote_current_release} + % { + 'user': env.remote_owner, + 'group': env.remote_group, + 'project_dir': env.remote_current_release, + } ) # symlink with new release fabric.api.execute(symlink) @@ -200,11 +239,16 @@ def rollback_code(): # elif nb_releases == 1: elif nb_releases == 1: - fabric.api.puts(fabric.colors.red('No rollback only one release found on remote !')) + fabric.api.puts( + fabric.colors.red('No rollback only one release found on remote !') + ) else: fabric.api.sudo( "rm %(current_path)s && rm -rf %(previous_release)s" - % {'current_path': env.remote_current_path, 'previous_release': env.remote_current_release} + % { + 'current_path': env.remote_current_path, + 'previous_release': env.remote_current_release, + } ) @@ -217,7 +261,10 @@ def symlink(): # TODO : really usefull ? (eg : for php apps ...) fabric.api.sudo( "ln -nfs %(shared_path)s/log %(current_release)s/log" - % {'shared_path': env.remote_shared_path, 'current_release': env.remote_current_release} + % { + 'shared_path': env.remote_shared_path, + 'current_release': env.remote_current_release, + } ) if env.has_key('cfg_shared_files'): @@ -249,7 +296,11 @@ def run_tests(): # Runs local unit test authorized_commands = ['tox'] if env.run_tests_command in authorized_commands: - with fabric.api.lcd('%s/%s-%s/' % (env.local_tmp_dir, env.application_name, env.tag.lower())): + with fabric.api.lcd( + '%s/%s-%s/' % (env.local_tmp_dir, env.application_name, env.tag.lower()) + ): fabric.api.local(env.run_tests_command) else: - fabric.api.abort(fabric.colors.red("wrong test command. Currently, only tox is supported")) + fabric.api.abort( + fabric.colors.red("wrong test command. Currently, only tox is supported") + ) diff --git a/pydiploy/require/system.py b/pydiploy/require/system.py index 50a4c19..2dc4264 100644 --- a/pydiploy/require/system.py +++ b/pydiploy/require/system.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -This class is for sytem relatives tools and commands +This class is for system relatives tools and commands """ @@ -113,8 +113,6 @@ def check_python3_install(version='python3', update=False): if not package_installed(version): # TODO check for others ubuntu"s versions !!!!! - fabric.api.puts(fabtools.system.distrib_release()) - fabric.api.puts(fabtools.system.distrib_release() >= 14.04) # Install mighty PPA if fabtools.system.distrib_release() >= 14.04: # add-apt-repository moved to software-properties-common in 14.04 diff --git a/pydiploy/scripts.py b/pydiploy/scripts.py index 9822aae..f33005a 100644 --- a/pydiploy/scripts.py +++ b/pydiploy/scripts.py @@ -16,17 +16,19 @@ SUBLIME_SNIPPET_FILENAME = 'pydiployfabfile.sublime-snippet' # TODO manage other os (Darwin,windoz...) SUBLIME_CONFIG_DIRS = { - 'Linux' : [ + 'Linux': [ '.config/sublime-text-3/Packages/User', - '.config/sublime-text-2/Packages/User' + '.config/sublime-text-2/Packages/User', ], } def get_dest_files(system): try: - return (join(expanduser('~'), cfg_dir, SUBLIME_SNIPPET_FILENAME) - for cfg_dir in SUBLIME_CONFIG_DIRS[system]) + return ( + join(expanduser('~'), cfg_dir, SUBLIME_SNIPPET_FILENAME) + for cfg_dir in SUBLIME_CONFIG_DIRS[system] + ) except KeyError: raise NotImplementedError(system) @@ -38,7 +40,10 @@ def sublime_text_snippet(): stream = StringIO() snippet_header = '\n\nPydiploy fabfile %s\npydiployfab\nsource.python\n' % __version__ + snippet_footer = ( + '\n]]>\nPydiploy fabfile %s\npydiployfab\nsource.python\n' + % __version__ + ) stream.write(snippet_header) with open(src_filename, 'r') as src_h: diff --git a/pydiploy/simple.py b/pydiploy/simple.py index 9eeaa9c..7f2a42a 100644 --- a/pydiploy/simple.py +++ b/pydiploy/simple.py @@ -14,8 +14,9 @@ def wrap_deploy(): yield except SystemExit: fabric.api.execute(rollback) - fabric.api.abort(fabric.colors.red( - "Deploy failed rollbacking process launched")) + fabric.api.abort( + fabric.colors.red("Deploy failed rollbacking process launched") + ) @do_verbose @@ -23,21 +24,23 @@ def application_packages(update=False): """ Installs all packages for the app """ fabtools.require.deb.packages(['gettext'], update=update) if env.remote_python_version >= 3: - fabric.api.execute(pydiploy.require.system.check_python3_install, - version='python%s' % env.remote_python_version) + fabric.api.execute( + pydiploy.require.system.check_python3_install, + version='python%s' % env.remote_python_version, + ) fabric.api.execute(pydiploy.require.python.utils.python_pkg) if 'extra_ppa_to_install' in env: fabric.api.execute( - pydiploy.require.system.install_extra_ppa, - env.extra_ppa_to_install) + pydiploy.require.system.install_extra_ppa, env.extra_ppa_to_install + ) if 'extra_source_to_install' in env: fabric.api.execute( - pydiploy.require.system.install_extra_source, - env.extra_source_to_install) + pydiploy.require.system.install_extra_source, env.extra_source_to_install + ) if 'extra_pkg_to_install' in env: fabric.api.execute( - pydiploy.require.system.install_extra_packages, - env.extra_pkg_to_install) + pydiploy.require.system.install_extra_packages, env.extra_pkg_to_install + ) def pre_install_backend(commands='/usr/bin/rsync'): @@ -56,11 +59,10 @@ def deploy_backend(upgrade_pkg=False, **kwargs): fabric.api.execute(pydiploy.require.releases_manager.setup) fabric.api.execute(pydiploy.require.releases_manager.deploy_code) fabric.api.execute( - pydiploy.require.python.utils.application_dependencies, - upgrade_pkg) + pydiploy.require.python.utils.application_dependencies, upgrade_pkg + ) # TODO PUT THIS METHOD IN OTHER PACKAGE - fabric.api.execute(pydiploy.require.simple.utils.app_settings, - **kwargs) + fabric.api.execute(pydiploy.require.simple.utils.app_settings, **kwargs) fabric.api.execute(pydiploy.require.simple.utils.deploy_environ_file) fabric.api.execute(pydiploy.require.system.permissions) fabric.api.execute(pydiploy.require.releases_manager.cleanup) @@ -71,9 +73,6 @@ def post_install_backend(): pass - - - def rollback(): """ Rollback code (current-1 release). """ fabric.api.execute(pydiploy.require.releases_manager.rollback_code) diff --git a/pydiploy/version.py b/pydiploy/version.py index 85aa75d..df82f5f 100644 --- a/pydiploy/version.py +++ b/pydiploy/version.py @@ -1,3 +1,3 @@ # pydiploy version __version__ = '1.1.7.9' -__version_info__ = tuple([ int(num) for num in __version__.split('.')]) +__version_info__ = tuple([int(num) for num in __version__.split('.')]) diff --git a/tests/functional_tests/conftest.py b/tests/functional_tests/conftest.py index cd2d8c0..1ba9e2e 100644 --- a/tests/functional_tests/conftest.py +++ b/tests/functional_tests/conftest.py @@ -1,16 +1,13 @@ -from pipes import quote import logging import os import sys +from pipes import quote -from mock import patch import pytest - from fabric.api import env, hide, lcd, local, settings from fabric.state import connections - from fabtools.vagrant import version as _vagrant_version - +from mock import patch HERE = os.path.dirname(__file__) @@ -19,7 +16,9 @@ def setup_package(request): vagrant_box = os.environ.get('FABTOOLS_TEST_BOX') if not vagrant_box: - pytest.skip("Set FABTOOLS_TEST_BOX to choose a Vagrant base box for functional tests") + pytest.skip( + "Set FABTOOLS_TEST_BOX to choose a Vagrant base box for functional tests" + ) vagrant_provider = os.environ.get('FABTOOLS_TEST_PROVIDER') reuse_vm = os.environ.get('FABTOOLS_TEST_REUSE_VM') _check_vagrant_version() @@ -42,7 +41,10 @@ def _check_vagrant_version(): if VAGRANT_VERSION is None: pytest.skip("Vagrant is required for functional tests") elif VAGRANT_VERSION < MIN_VAGRANT_VERSION: - pytest.skip("Vagrant >= %s is required for functional tests" % ".".join(map(str, MIN_VAGRANT_VERSION))) + pytest.skip( + "Vagrant >= %s is required for functional tests" + % ".".join(map(str, MIN_VAGRANT_VERSION)) + ) def _configure_logging(): @@ -123,7 +125,9 @@ def _clear_fabric_connection_cache(): def _update_package_index(): from fabtools.system import distrib_family + family = distrib_family() if family == 'debian': from fabtools.require.deb import uptodate_index + uptodate_index() diff --git a/tests/functional_tests/test_pre_install_backend.py b/tests/functional_tests/test_pre_install_backend.py index 5d887a1..ca29427 100644 --- a/tests/functional_tests/test_pre_install_backend.py +++ b/tests/functional_tests/test_pre_install_backend.py @@ -1,8 +1,8 @@ -import pytest -from fabric.api import run, env -import pydiploy import os +import pydiploy +import pytest +from fabric.api import env, run pytestmark = pytest.mark.network @@ -13,14 +13,17 @@ def buildmyenv(): env.remote_owner = 'django' # remote server user env.remote_group = 'di' # remote server group - env.application_name = 'myapp' # name of webapp + env.application_name = 'myapp' # name of webapp env.root_package_name = 'myapp' # name of app in webapp env.remote_home = '/home/django' # remote home root env.remote_python_version = 3.4 # python version - env.remote_virtualenv_root = os.path.join(env.remote_home, '.virtualenvs') # venv root - env.remote_virtualenv_dir = os.path.join(env.remote_virtualenv_root, - env.application_name) # venv for webapp dir + env.remote_virtualenv_root = os.path.join( + env.remote_home, '.virtualenvs' + ) # venv root + env.remote_virtualenv_dir = os.path.join( + env.remote_virtualenv_root, env.application_name + ) # venv for webapp dir env.remote_repo_url = 'git@git.net:myapp.git' # git repository url env.local_tmp_dir = '/tmp' # tmp dir env.remote_static_root = '/var/www/static' # root of static files @@ -60,8 +63,10 @@ def pre_install(): def test_user_created(pre_install): django = run('getent passwd %s' % env.remote_owner) - assert django == '%s:x:1001:1003::/home/%s:/bin/bash' % (env.remote_owner, - env.remote_owner) + assert django == '%s:x:1001:1003::/home/%s:/bin/bash' % ( + env.remote_owner, + env.remote_owner, + ) django = run('getent group %s' % env.remote_group) assert django == '%s:x:1003:' % env.remote_group @@ -73,7 +78,9 @@ def test_locale(pre_install): def test_timezone(pre_install): - lang = run("cd /usr/share/zoneinfo && find * -type f -exec sh -c \"diff -q /etc/localtime '{}' > /dev/null && echo {}\" \;") + lang = run( + "cd /usr/share/zoneinfo && find * -type f -exec sh -c \"diff -q /etc/localtime '{}' > /dev/null && echo {}\" \;" + ) assert lang == env.timezone @@ -88,7 +95,9 @@ def test_app_packages(pre_install): assert install == 'install ok installed' install = run("dpkg-query -W -f='${Status}' python-pip") assert install == 'install ok installed' - install = run("dpkg-query -W -f='${Status}' python%s-dev" % env.remote_python_version) + install = run( + "dpkg-query -W -f='${Status}' python%s-dev" % env.remote_python_version + ) assert install == 'install ok installed' @@ -96,9 +105,13 @@ def test_circus(pre_install): install = run("pip freeze |grep circus |cut -d '=' -f1") assert install == 'circus\r\ncircus-web' - circusconf = run('[ -e %s/.circus.d ] && echo "Found" || echo "Not found"' % env.remote_home) + circusconf = run( + '[ -e %s/.circus.d ] && echo "Found" || echo "Not found"' % env.remote_home + ) assert circusconf == "Found" - circusconf = run('[ -e %s/.circus.ini ] && echo "Found" || echo "Not found"' % env.remote_home) + circusconf = run( + '[ -e %s/.circus.ini ] && echo "Found" || echo "Not found"' % env.remote_home + ) assert circusconf == "Found" start = run("status circus |cut -d '/' -f1") @@ -106,5 +119,8 @@ def test_circus(pre_install): def test_virtualenv(pre_install): - right_lib = run('[ -e %s/.virtualenvs/%s/lib/python%s ] && echo "Found" || echo "Not found"' % (env.remote_home, env.application_name, env.remote_python_version)) + right_lib = run( + '[ -e %s/.virtualenvs/%s/lib/python%s ] && echo "Found" || echo "Not found"' + % (env.remote_home, env.application_name, env.remote_python_version) + ) assert right_lib == "Found" diff --git a/tests/functional_tests/test_pre_install_frontend.py b/tests/functional_tests/test_pre_install_frontend.py index 4fe28a9..dc517d8 100644 --- a/tests/functional_tests/test_pre_install_frontend.py +++ b/tests/functional_tests/test_pre_install_frontend.py @@ -1,8 +1,8 @@ -import pytest -from fabric.api import run, env -import pydiploy import os +import pydiploy +import pytest +from fabric.api import env, run pytestmark = pytest.mark.network @@ -13,14 +13,17 @@ def buildmyenv(): env.remote_owner = 'django' # remote server user env.remote_group = 'di' # remote server group - env.application_name = 'myapp' # name of webapp + env.application_name = 'myapp' # name of webapp env.root_package_name = 'myapp' # name of app in webapp env.remote_home = '/home/django' # remote home root env.remote_python_version = 3.4 # python version - env.remote_virtualenv_root = os.path.join(env.remote_home, '.virtualenvs') # venv root - env.remote_virtualenv_dir = os.path.join(env.remote_virtualenv_root, - env.application_name) # venv for webapp dir + env.remote_virtualenv_root = os.path.join( + env.remote_home, '.virtualenvs' + ) # venv root + env.remote_virtualenv_dir = os.path.join( + env.remote_virtualenv_root, env.application_name + ) # venv for webapp dir env.remote_repo_url = 'git@git.net:myapp.git' # git repository url env.local_tmp_dir = '/tmp' # tmp dir env.remote_static_root = '/var/www/static' # root of static files diff --git a/tests/test_bottle.py b/tests/test_bottle.py index a8bd1da..f65b008 100644 --- a/tests/test_bottle.py +++ b/tests/test_bottle.py @@ -5,15 +5,26 @@ from unittest import TestCase from fabric.api import env -from mock import call, Mock, patch -from pydiploy.bottle import (application_packages, - deploy_backend, deploy_frontend, - install_oracle_client, install_postgres_server, - install_sap_client, post_install_backend, - post_install_frontend, pre_install_backend, - pre_install_frontend, reload_backend, - reload_frontend, rollback, set_app_down, - set_app_up, wrap_deploy) +from mock import Mock, call, patch + +from pydiploy.bottle import ( + application_packages, + deploy_backend, + deploy_frontend, + install_oracle_client, + install_postgres_server, + install_sap_client, + post_install_backend, + post_install_frontend, + pre_install_backend, + pre_install_frontend, + reload_backend, + reload_frontend, + rollback, + set_app_down, + set_app_up, + wrap_deploy, +) class ReleasesManagerCheck(TestCase): @@ -41,98 +52,148 @@ def test_wrap_deploy(self, api_execute, api_abort): def test_application_packages(self, api_execute, deb_packages): application_packages() self.assertTrue(deb_packages.called) - self.assertEqual( - deb_packages.call_args, call(['gettext'], update=False)) + self.assertEqual(deb_packages.call_args, call(['gettext'], update=False)) self.assertTrue(api_execute.called) self.assertTrue( - str(api_execute.call_args_list[0]).find('call(= 3 env.remote_python_version = 3 application_packages() self.assertTrue(api_execute.called) self.assertTrue( - str(api_execute.call_args_list[1]).find('call(= 3 env.remote_python_version = 3 application_packages() self.assertTrue(api_execute.called) self.assertTrue( - str(api_execute.call_args_list[1]).find('call( 0) + self.assertTrue(str(api_execute.call_args).find("function permissions") > 0) self.assertTrue(api_sudo.called) self.assertEqual( - api_sudo.call_args_list, [call( - 'mkdir -p remote_project_dir/{releases,shared}'), - call('mkdir -p remote_shared_path/{config,log}')]) + api_sudo.call_args_list, + [ + call('mkdir -p remote_project_dir/{releases,shared}'), + call('mkdir -p remote_shared_path/{config,log}'), + ], + ) # extra_symlinks_dirs provided env.extra_symlink_dirs = ['symdir'] setup() - self.assertTrue(str(api_sudo.call_args_list[4]).find( - "mkdir -p remote_shared_path/symdir") > 0) + self.assertTrue( + str(api_sudo.call_args_list[4]).find("mkdir -p remote_shared_path/symdir") + > 0 + ) @patch('fabric.api.sudo', return_value=Mock()) @patch('pydiploy.prepare.process_releases', return_value=Mock()) def test_cleanup(self, rel_manager, api_sudo): cleanup() self.assertTrue(api_sudo.called) - self.assertEqual( - api_sudo.call_args, call('rm -rf remote_releases_path/1.0')) + self.assertEqual(api_sudo.call_args, call('rm -rf remote_releases_path/1.0')) @patch('pydiploy.require.git.check_tag_exist', return_value=Mock()) @patch('fabric.api.prompt', return_value=Mock()) @@ -98,7 +117,21 @@ def test_cleanup(self, rel_manager, api_sudo): @patch('fabric.contrib.project.rsync_project', return_value=Mock()) @patch('fabtools.files.is_file', return_value=None) @patch('os.path.exists', return_value=False) - def test_deploy_code(self, path_exists, is_file, rsync_project, git_archive, upload_template, api_execute, api_sudo, api_lcd, api_require, api_local, api_prompt, tag_exist): + def test_deploy_code( + self, + path_exists, + is_file, + rsync_project, + git_archive, + upload_template, + api_execute, + api_sudo, + api_lcd, + api_require, + api_local, + api_prompt, + tag_exist, + ): api_lcd.return_value.__exit__ = Mock() api_lcd.return_value.__enter__ = Mock() @@ -106,62 +139,96 @@ def test_deploy_code(self, path_exists, is_file, rsync_project, git_archive, upl self.assertTrue(rsync_project.called) self.assertTrue( - str(rsync_project.call_args).find("'/tmp/appliname-mytag/'") > 0) - self.assertTrue(str(rsync_project.call_args).find( - "extra_opts='--links --rsync-path=\"sudo -u remote_owner rsync\"'") > 0) + str(rsync_project.call_args).find("'/tmp/appliname-mytag/'") > 0 + ) + self.assertTrue( + str(rsync_project.call_args).find( + "extra_opts='--links --rsync-path=\"sudo -u remote_owner rsync\"'" + ) + > 0 + ) self.assertTrue(str(rsync_project.call_args).find("delete=True") > 0) - self.assertTrue(str(rsync_project.call_args).find( - "exclude=['fabfile', 'MANIFEST.in', '*.ignore', 'docs', '*.log', 'bin', 'manage.py', '.tox', 'root_package_name/wsgi.py', '*.db', '.gitignore', '.gitattributes', 'root_package_name/settings/dev.py', 'root_package_name/settings/test.py', 'root_package_name/settings/prod.py'") > 0) + self.assertTrue( + str(rsync_project.call_args).find( + "exclude=['fabfile', 'MANIFEST.in', '*.ignore', 'docs', '*.log', 'bin', 'manage.py', '.tox', 'root_package_name/wsgi.py', '*.db', '.gitignore', '.gitattributes', 'root_package_name/settings/dev.py', 'root_package_name/settings/test.py', 'root_package_name/settings/prod.py'" + ) + > 0 + ) self.assertTrue(git_archive.called) - self.assertEqual(git_archive.call_args, call( - 'appliname', prefix='appliname-mytag/', tag='mytag', remote='remote_repo_url', specific_folder='mysubfolder')) + self.assertEqual( + git_archive.call_args, + call( + 'appliname', + prefix='appliname-mytag/', + tag='mytag', + remote='remote_repo_url', + specific_folder='mysubfolder', + ), + ) self.assertTrue(upload_template.called) - self.assertTrue(str(upload_template.call_args_list[0]).find( - "'/tmp/appliname-mytag/README'") > 0) - self.assertTrue(str(upload_template.call_args_list[0]).find( - "'remote_shared_path/config'") > 0) + self.assertTrue( + str(upload_template.call_args_list[0]).find("'/tmp/appliname-mytag/README'") + > 0 + ) + self.assertTrue( + str(upload_template.call_args_list[0]).find("'remote_shared_path/config'") + > 0 + ) self.assertTrue(api_execute.called) - self.assertTrue(str(api_execute.call_args_list[0]).find( - "function symlink") > 0) - self.assertTrue(str(api_execute.call_args_list[1]).find( - "function set_current") > 0) + self.assertTrue(str(api_execute.call_args_list[0]).find("function symlink") > 0) + self.assertTrue( + str(api_execute.call_args_list[1]).find("function set_current") > 0 + ) self.assertTrue(api_sudo.called) - self.assertTrue(str(api_sudo.call_args).find( - "chown -R remote_owner:remote_group remote_releases_path/") > 0) + self.assertTrue( + str(api_sudo.call_args).find( + "chown -R remote_owner:remote_group remote_releases_path/" + ) + > 0 + ) self.assertTrue(api_lcd.called) - self.assertTrue(str(api_lcd.call_args_list[1]).find('/tmp/appliname-mytag/') > 0) - #self.assertTrue(str(api_lcd.call_args_list[2]).find('rm myarchive') > 0 ) - #self.assertTrue(str(api_lcd.call_args_list[3]).find('rm -rf /tmp/appliname-mytag') > 0 ) + self.assertTrue( + str(api_lcd.call_args_list[1]).find('/tmp/appliname-mytag/') > 0 + ) + # self.assertTrue(str(api_lcd.call_args_list[2]).find('rm myarchive') > 0 ) + # self.assertTrue(str(api_lcd.call_args_list[3]).find('rm -rf /tmp/appliname-mytag') > 0 ) self.assertTrue(api_local.called) self.assertTrue(str(api_local.call_args_list[0]).find('tar xvf myarchive')) self.assertTrue(str(api_local.call_args_list[1]).find(env.run_tests_command)) self.assertTrue(api_require.called) self.assertEqual( - api_require.call_args_list, [call( - 'tag', provided_by=['tag', 'head']), - call('remote_project_dir', provided_by=['dev', 'test', 'prod'])]) + api_require.call_args_list, + [ + call('tag', provided_by=['tag', 'head']), + call('remote_project_dir', provided_by=['dev', 'test', 'prod']), + ], + ) self.assertTrue(is_file.called) - self.assertEqual(is_file.call_args, call( - path='remote_shared_path/config/README', use_sudo=True)) + self.assertEqual( + is_file.call_args, + call(path='remote_shared_path/config/README', use_sudo=True), + ) # extra_symlink_dirs provided - env.extra_symlink_dirs = ['symdir', ] + env.extra_symlink_dirs = [ + 'symdir', + ] deploy_code() del env['tag'] - tag_exist.side_effect = [False,True] - api_prompt.return_value='4.0' + tag_exist.side_effect = [False, True] + api_prompt.return_value = '4.0' deploy_code() self.assertTrue(api_prompt.called) self.assertEqual(env.tag, '4.0') - path_exists.return_value=True + path_exists.return_value = True deploy_code() self.assertTrue(api_local.called) @@ -174,35 +241,51 @@ def test_rollback_code(self, process_rel, api_sudo, api_puts): env.releases = [] rollback_code() self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, - call('rm remote_current_path && rm -rf remote_current_release')) + self.assertEqual( + api_sudo.call_args, + call('rm remote_current_path && rm -rf remote_current_release'), + ) # one release env.releases = ["1.0"] rollback_code() self.assertTrue(api_puts.called) - self.assertEqual(api_puts.call_args, - call('rollback_code : \x1b[32mDone\x1b[0m')) + self.assertEqual( + api_puts.call_args, call('rollback_code : \x1b[32mDone\x1b[0m') + ) env.releases = ["1.0", "2.0", "3.0", "4.0"] rollback_code() self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, - call('rm remote_current_path; ln -s 3.0 remote_current_path && rm -rf remote_releases_path/4.0')) - + self.assertEqual( + api_sudo.call_args, + call( + 'rm remote_current_path; ln -s 3.0 remote_current_path && rm -rf remote_releases_path/4.0' + ), + ) @patch('fabric.api.sudo', return_value=Mock()) def test_symlink(self, api_sudo): symlink() self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, - call('ln -nfs remote_shared_path/config/README remote_current_release/README')) + self.assertEqual( + api_sudo.call_args, + call( + 'ln -nfs remote_shared_path/config/README remote_current_release/README' + ), + ) # extra_symlink_dirs provided - env.extra_symlink_dirs = ['symdir', ] + env.extra_symlink_dirs = [ + 'symdir', + ] symlink() - self.assertTrue(str(api_sudo.call_args_list[4]).find( - "ln -nfs remote_shared_path/symdir remote_current_release/symdir") > 0) + self.assertTrue( + str(api_sudo.call_args_list[4]).find( + "ln -nfs remote_shared_path/symdir remote_current_release/symdir" + ) + > 0 + ) @patch('fabric.api.abort', return_value=Mock()) @patch('fabric.api.local', return_value=Mock()) diff --git a/tests/test_require_bottle.py b/tests/test_require_bottle.py index ddcc6a6..0e5ccd8 100644 --- a/tests/test_require_bottle.py +++ b/tests/test_require_bottle.py @@ -4,10 +4,14 @@ import copy import datetime from unittest import TestCase + from fabric.api import env -from mock import call, Mock, patch -from pydiploy.require.bottle.utils import (app_settings, deploy_environ_file, - extract_settings) +from mock import Mock, call, patch +from pydiploy.require.bottle.utils import ( + app_settings, + deploy_environ_file, + extract_settings, +) class UtilsCheck(TestCase): @@ -24,8 +28,11 @@ def setUp(self): env.map_settings = { "server_url": "SERVER_URL", "default_db_user": "DATABASES['default']['USER']", - 'thanks_arno': ('THANKS_ARNAUD', - r'environ.get\(["\']THANKS_ARNAUD[\'"], [\'"](.*)["\']\)')} + 'thanks_arno': ( + 'THANKS_ARNAUD', + r'environ.get\(["\']THANKS_ARNAUD[\'"], [\'"](.*)["\']\)', + ), + } env.local_tmp_root_app_package = "local_tmp_root_app_package" env.remote_owner = "owner" env.previous_settings_file = "remote_settings_file" @@ -43,8 +50,9 @@ def test_extract_settings(self, api_get): extract_settings() self.assertTrue(api_get.called) - self.assertEqual(api_get.call_args, call( - 'remote_settings_file', local_path='tests/data')) + self.assertEqual( + api_get.call_args, call('remote_settings_file', local_path='tests/data') + ) self.assertEqual(env.server_url, '{{ server_url }}') self.assertEqual(env.default_db_user, '{{ default_db_user }}') @@ -59,12 +67,12 @@ def test_app_settings(self, upload_template, api_require, api_execute, is_file): app_settings(test1='toto') self.assertTrue(is_file.called) - self.assertEqual(is_file.call_args, call( - path='remote_settings_file', use_sudo=True)) + self.assertEqual( + is_file.call_args, call(path='remote_settings_file', use_sudo=True) + ) self.assertTrue(api_execute.called) - self.assertTrue( - str(api_execute.call_args).find('extract_settings') > 0) + self.assertTrue(str(api_execute.call_args).find('extract_settings') > 0) self.assertEqual(env.test1, 'toto') @@ -73,16 +81,18 @@ def test_app_settings(self, upload_template, api_require, api_execute, is_file): self.assertTrue(str(api_require.call_args).find('default_db_user') > 0) self.assertTrue(upload_template.called) - self.assertTrue(str(upload_template.call_args).find( - "template_dir='local_tmp_root_app_package/settings'") > 0) - self.assertTrue( - str(upload_template.call_args).find("'settings.py'") > 0) - self.assertTrue( - str(upload_template.call_args).find("'remote_settings_file'") > 0) self.assertTrue( - str(upload_template.call_args).find("use_jinja=True") > 0) + str(upload_template.call_args).find( + "template_dir='local_tmp_root_app_package/settings'" + ) + > 0 + ) + self.assertTrue(str(upload_template.call_args).find("'settings.py'") > 0) self.assertTrue( - str(upload_template.call_args).find("user='owner'") > 0) + str(upload_template.call_args).find("'remote_settings_file'") > 0 + ) + self.assertTrue(str(upload_template.call_args).find("use_jinja=True") > 0) + self.assertTrue(str(upload_template.call_args).find("user='owner'") > 0) # is file false is_file.return_value = False @@ -90,8 +100,9 @@ def test_app_settings(self, upload_template, api_require, api_execute, is_file): app_settings() self.assertTrue(is_file.called) - self.assertEqual(is_file.call_args, call( - path='remote_settings_file', use_sudo=True)) + self.assertEqual( + is_file.call_args, call(path='remote_settings_file', use_sudo=True) + ) @patch('fabtools.files.upload_template', return_value=Mock()) def test_deploy_environ_file(self, upload_template): diff --git a/tests/test_require_circus.py b/tests/test_require_circus.py index 94e1e50..1045b92 100644 --- a/tests/test_require_circus.py +++ b/tests/test_require_circus.py @@ -35,7 +35,14 @@ def tearDown(self): @patch('fabtools.files.upload_template', return_value=Mock()) @patch('fabtools.require.files.directory', return_value=Mock()) def test_circus_pkg_old( - self, files_directory, upload_template, python_install, deb_ppa, deb_packages, distrib_release, distrib_id + self, + files_directory, + upload_template, + python_install, + deb_ppa, + deb_packages, + distrib_release, + distrib_id, ): # no_circus_web = True @@ -46,7 +53,10 @@ def test_circus_pkg_old( self.assertTrue(distrib_id.called) self.assertTrue(deb_packages.called) - self.assertEqual(deb_packages.call_args_list, [call(['libzmq-dev', 'libevent-dev'], update=False)]) + self.assertEqual( + deb_packages.call_args_list, + [call(['libzmq-dev', 'libevent-dev'], update=False)], + ) # self.assertTrue(deb_ppa.called) # self.assertEqual(deb_ppa.call_args_list, [ @@ -54,26 +64,45 @@ def test_circus_pkg_old( self.assertTrue(python_install.called) - self.assertEqual(python_install.call_args_list, [call('circus', upgrade=False, use_sudo=True)]) + self.assertEqual( + python_install.call_args_list, + [call('circus', upgrade=False, use_sudo=True)], + ) self.assertTrue(upload_template.called) self.assertTrue(str(upload_template.call_args).find("'circus.ini.tpl'") > 0) - self.assertTrue(str(upload_template.call_args).find("'remote_home/.circus.ini'") > 0) - self.assertTrue(str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0) + self.assertTrue( + str(upload_template.call_args).find("'remote_home/.circus.ini'") > 0 + ) + self.assertTrue( + str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0 + ) self.assertTrue(str(upload_template.call_args).find("user='remote_owner'") > 0) self.assertTrue(files_directory.called) self.assertEqual( files_directory.call_args, - call(owner='remote_owner', path='remote_home/.circus.d', use_sudo=True, group='remote_group', mode='750'), + call( + owner='remote_owner', + path='remote_home/.circus.d', + use_sudo=True, + group='remote_group', + mode='750', + ), ) # test circus_package_name - env.circus_package_name = "https://github.com/githubaccount/circus/archive/master.zip" + env.circus_package_name = ( + "https://github.com/githubaccount/circus/archive/master.zip" + ) circus_pkg() self.assertEqual( python_install.call_args_list[1], - call('https://github.com/githubaccount/circus/archive/master.zip', upgrade=False, use_sudo=True), + call( + 'https://github.com/githubaccount/circus/archive/master.zip', + upgrade=False, + use_sudo=True, + ), ) # test no_circus_web @@ -110,7 +139,14 @@ def test_circus_pkg_old( @patch('fabtools.files.upload_template', return_value=Mock()) @patch('fabtools.require.files.directory', return_value=Mock()) def test_circus_pkg( - self, files_directory, upload_template, python_install, deb_ppa, deb_packages, distrib_release, distrib_id + self, + files_directory, + upload_template, + python_install, + deb_ppa, + deb_packages, + distrib_release, + distrib_id, ): # no_circus_web = True @@ -121,7 +157,10 @@ def test_circus_pkg( self.assertTrue(distrib_id.called) self.assertTrue(deb_packages.called) - self.assertEqual(deb_packages.call_args_list, [call(['libzmq3-dev', 'libevent-dev'], update=False)]) + self.assertEqual( + deb_packages.call_args_list, + [call(['libzmq3-dev', 'libevent-dev'], update=False)], + ) # self.assertTrue(deb_ppa.called) # self.assertEqual(deb_ppa.call_args_list, [ @@ -129,26 +168,45 @@ def test_circus_pkg( self.assertTrue(python_install.called) - self.assertEqual(python_install.call_args_list, [call('circus', upgrade=False, use_sudo=True)]) + self.assertEqual( + python_install.call_args_list, + [call('circus', upgrade=False, use_sudo=True)], + ) self.assertTrue(upload_template.called) self.assertTrue(str(upload_template.call_args).find("'circus.ini.tpl'") > 0) - self.assertTrue(str(upload_template.call_args).find("'remote_home/.circus.ini'") > 0) - self.assertTrue(str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0) + self.assertTrue( + str(upload_template.call_args).find("'remote_home/.circus.ini'") > 0 + ) + self.assertTrue( + str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0 + ) self.assertTrue(str(upload_template.call_args).find("user='remote_owner'") > 0) self.assertTrue(files_directory.called) self.assertEqual( files_directory.call_args, - call(owner='remote_owner', path='remote_home/.circus.d', use_sudo=True, group='remote_group', mode='750'), + call( + owner='remote_owner', + path='remote_home/.circus.d', + use_sudo=True, + group='remote_group', + mode='750', + ), ) # test circus_package_name - env.circus_package_name = "https://github.com/githubaccount/circus/archive/master.zip" + env.circus_package_name = ( + "https://github.com/githubaccount/circus/archive/master.zip" + ) circus_pkg() self.assertEqual( python_install.call_args_list[1], - call('https://github.com/githubaccount/circus/archive/master.zip', upgrade=False, use_sudo=True), + call( + 'https://github.com/githubaccount/circus/archive/master.zip', + upgrade=False, + use_sudo=True, + ), ) # test no_circus_web @@ -183,8 +241,15 @@ def test_app_circus_conf(self, upload_template): self.assertTrue(upload_template.called) self.assertTrue(str(upload_template.call_args).find("'app.ini.tpl'") > 0) - self.assertTrue(str(upload_template.call_args).find("'remote_home/.circus.d/application_name.ini'") > 0) - self.assertTrue(str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0) + self.assertTrue( + str(upload_template.call_args).find( + "'remote_home/.circus.d/application_name.ini'" + ) + > 0 + ) + self.assertTrue( + str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0 + ) self.assertTrue(str(upload_template.call_args).find("user='remote_owner'") > 0) @patch('fabtools.files.is_dir', return_value=False) @@ -194,8 +259,12 @@ def test_upstart(self, upload_template, is_systemd): self.assertTrue(upload_template.called) self.assertTrue(str(upload_template.call_args).find("'upstart.conf.tpl'") > 0) - self.assertTrue(str(upload_template.call_args).find("'/etc/init/circus.conf'") > 0) - self.assertTrue(str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0) + self.assertTrue( + str(upload_template.call_args).find("'/etc/init/circus.conf'") > 0 + ) + self.assertTrue( + str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0 + ) self.assertTrue(str(upload_template.call_args).find("user='root'") > 0) @patch('fabric.api.sudo', return_value=Mock()) @@ -206,8 +275,13 @@ def test_upstart_systemd(self, upload_template, is_systemd, mock_reload_daemon): self.assertTrue(upload_template.called) self.assertTrue(str(upload_template.call_args).find("'circus.service.tpl'") > 0) - self.assertTrue(str(upload_template.call_args).find("'/etc/systemd/system/circus.service'") > 0) - self.assertTrue(str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0) + self.assertTrue( + str(upload_template.call_args).find("'/etc/systemd/system/circus.service'") + > 0 + ) + self.assertTrue( + str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0 + ) self.assertTrue(str(upload_template.call_args).find("user='root'") > 0) self.assertTrue(mock_reload_daemon.called) @@ -224,7 +298,8 @@ def test_app_reload(self, api_sudo, api_settings, is_systemd): self.assertTrue(api_sudo.called) self.assertEqual( - api_sudo.call_args_list, [call('status circus'), call('circusctl reloadconfig')], + api_sudo.call_args_list, + [call('status circus'), call('circusctl reloadconfig')], ) self.assertTrue(api_settings.called) diff --git a/tests/test_require_databases.py b/tests/test_require_databases.py index 92fdcdc..50c3e40 100644 --- a/tests/test_require_databases.py +++ b/tests/test_require_databases.py @@ -5,22 +5,28 @@ from unittest import TestCase from fabric.api import env -from mock import call, Mock, patch +from mock import Mock, call, patch from pydiploy.require.databases.ldap import ldap_pkg from pydiploy.require.databases.mongodb import install_mongodb -from pydiploy.require.databases.mysql import (add_mysql_database, - add_mysql_user, - install_mysql_client, - install_mysql_server) -from pydiploy.require.databases.oracle import (get_oracle_jdk_version, - install_oracle_client, - install_oracle_jdk) -from pydiploy.require.databases.postgres import (add_postgres_database, - add_postgres_user, - install_postgres_server, - postgres_pkg) -from pydiploy.require.databases.sqlite import sqlite3_pkg +from pydiploy.require.databases.mysql import ( + add_mysql_database, + add_mysql_user, + install_mysql_client, + install_mysql_server, +) +from pydiploy.require.databases.oracle import ( + get_oracle_jdk_version, + install_oracle_client, + install_oracle_jdk, +) +from pydiploy.require.databases.postgres import ( + add_postgres_database, + add_postgres_user, + install_postgres_server, + postgres_pkg, +) from pydiploy.require.databases.sap import install_sap_client +from pydiploy.require.databases.sqlite import sqlite3_pkg class LdapCheck(TestCase): @@ -43,12 +49,20 @@ def test_ldap_pkg(self, python_package, deb_package): ldap_pkg() self.assertTrue(python_package.called) - self.assertEqual(python_package.call_args, - call('python-ldap', upgrade=True, use_sudo=False, user=None)) + self.assertEqual( + python_package.call_args, + call('python-ldap', upgrade=True, use_sudo=False, user=None), + ) self.assertTrue(deb_package.called) - self.assertEqual(deb_package.call_args_list, - [call('libldap2-dev', update=True), call('libsasl2-dev', update=True), call('libssl-dev', update=True)]) + self.assertEqual( + deb_package.call_args_list, + [ + call('libldap2-dev', update=True), + call('libsasl2-dev', update=True), + call('libssl-dev', update=True), + ], + ) class Sqlite3Check(TestCase): @@ -71,12 +85,13 @@ def test_sqlite3_pkg(self, python_package, deb_package): sqlite3_pkg() self.assertTrue(python_package.called) - self.assertEqual(python_package.call_args, - call('pysqlite', upgrade=True, use_sudo=False, user=None)) + self.assertEqual( + python_package.call_args, + call('pysqlite', upgrade=True, use_sudo=False, user=None), + ) self.assertTrue(deb_package.called) - self.assertEqual(deb_package.call_args, - call('libsqlite3-dev', update=True)) + self.assertEqual(deb_package.call_args, call('libsqlite3-dev', update=True)) class PostgresCheck(TestCase): @@ -100,8 +115,7 @@ def test_postgres_pkg(self, deb_packages): postgres_pkg() self.assertTrue(deb_packages.called) - self.assertEqual( - deb_packages.call_args, call(['libpq-dev'], update=False)) + self.assertEqual(deb_packages.call_args, call(['libpq-dev'], update=False)) @patch("fabtools.require.postgres.server", return_value=Mock()) def test_install_postgres_server(self, postgres_server): @@ -116,7 +130,9 @@ def test_add_postgres_user(self, postgres_user, postgres_user_exist): add_postgres_user(name='bill', password='g@t3s') self.assertTrue(postgres_user.called) self.assertEqual( - postgres_user.call_args, call('bill', 'g@t3s', False, False, False, True, True, None, False)) + postgres_user.call_args, + call('bill', 'g@t3s', False, False, False, True, True, None, False), + ) # verbose_output = false env.verbose_output = False @@ -127,7 +143,6 @@ def test_add_postgres_user(self, postgres_user, postgres_user_exist): postgres_user_exist.return_value = True add_postgres_user(name='bill', password='g@t3s') - @patch("fabtools.postgres.database_exists", return_value=False) @patch("fabtools.require.postgres.database", return_value=Mock()) def test_add_postgres_database(self, postgres_database, postgres_db_exists): @@ -163,8 +178,9 @@ def test_install_mysql_client(self, deb_packages): install_mysql_client() self.assertTrue(deb_packages.called) - self.assertEqual(deb_packages.call_args, - call(['libmysqlclient-dev'], update=False)) + self.assertEqual( + deb_packages.call_args, call(['libmysqlclient-dev'], update=False) + ) @patch("fabtools.require.mysql.server", return_value=Mock()) def test_install_mysql_server(self, mysql_server): @@ -220,9 +236,11 @@ def setUp(self): env.oracle_client_version = '11.2' env.oracle_download_url = 'http://librepo.net/lib/oracle/' env.oracle_remote_dir = 'oracle_client' - env.oracle_packages = ['instantclient-basic-linux-x86-64-11.2.0.2.0.zip', - 'instantclient-sdk-linux-x86-64-11.2.0.2.0.zip', - 'instantclient-sqlplus-linux-x86-64-11.2.0.2.0.zip'] + env.oracle_packages = [ + 'instantclient-basic-linux-x86-64-11.2.0.2.0.zip', + 'instantclient-sdk-linux-x86-64-11.2.0.2.0.zip', + 'instantclient-sqlplus-linux-x86-64-11.2.0.2.0.zip', + ] env.locale = 'fr_FR.UTF-8' def tearDown(self): @@ -235,8 +253,9 @@ def tearDown(self): @patch("fabric.api.cd", return_value=Mock()) @patch("fabtools.require.files.directory", return_value=Mock()) @patch("fabtools.require.deb.packages", return_value=Mock()) - def test_install_oracle_client(self, deb_packages, files_directory, - api_cd, api_sudo, files_is_link, api_abort): + def test_install_oracle_client( + self, deb_packages, files_directory, api_cd, api_sudo, files_is_link, api_abort + ): api_cd.return_value.__exit__ = Mock() api_cd.return_value.__enter__ = Mock() @@ -244,45 +263,65 @@ def test_install_oracle_client(self, deb_packages, files_directory, install_oracle_client() self.assertTrue(deb_packages.called) - self.assertEqual(deb_packages.call_args, - call(['libaio-dev', 'unzip'])) + self.assertEqual(deb_packages.call_args, call(['libaio-dev', 'unzip'])) self.assertTrue(files_directory.called) - self.assertEqual(files_directory.call_args, - call(path='/home/django/oracle_client', use_sudo=True, owner='django', - group='di', mode='750')) + self.assertEqual( + files_directory.call_args, + call( + path='/home/django/oracle_client', + use_sudo=True, + owner='django', + group='di', + mode='750', + ), + ) self.assertTrue(api_cd.called) - self.assertEqual(api_cd.call_args_list, - [call('/home/django'), call('oracle_client/instantclient_11_2')]) + self.assertEqual( + api_cd.call_args_list, + [call('/home/django'), call('oracle_client/instantclient_11_2')], + ) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args_list, - [call('wget -c http://librepo.net/lib/oracle/instantclient-basic-linux-x86-64-11.2.0.2.0.zip'), - call( - 'unzip instantclient-basic-linux-x86-64-11.2.0.2.0.zip -d oracle_client'), - call( - 'rm /home/django/instantclient-basic-linux-x86-64-11.2.0.2.0.zip'), - call( - 'wget -c http://librepo.net/lib/oracle/instantclient-sdk-linux-x86-64-11.2.0.2.0.zip'), - call( - 'unzip instantclient-sdk-linux-x86-64-11.2.0.2.0.zip -d oracle_client'), - call( - 'rm /home/django/instantclient-sdk-linux-x86-64-11.2.0.2.0.zip'), - call( - 'wget -c http://librepo.net/lib/oracle/instantclient-sqlplus-linux-x86-64-11.2.0.2.0.zip'), - call( - 'unzip instantclient-sqlplus-linux-x86-64-11.2.0.2.0.zip -d oracle_client'), - call( - 'rm /home/django/instantclient-sqlplus-linux-x86-64-11.2.0.2.0.zip'), - call('ln -s libclntsh.so.* libclntsh.so'), - call('pwd'), - call( - "echo '# ORACLE CLIENT CONFIGURATION\nexport ORACLE_HOME=/home/django/oracle_client/instantclient_11_2\nexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ORACLE_HOME' >> .bashrc"), - call('source .bashrc'), - call( - 'echo /home/django/oracle_client/instantclient_11_2 > /etc/ld.so.conf.d/oracle.conf'), - call('ldconfig')]) + self.assertEqual( + api_sudo.call_args_list, + [ + call( + 'wget -c http://librepo.net/lib/oracle/instantclient-basic-linux-x86-64-11.2.0.2.0.zip' + ), + call( + 'unzip instantclient-basic-linux-x86-64-11.2.0.2.0.zip -d oracle_client' + ), + call('rm /home/django/instantclient-basic-linux-x86-64-11.2.0.2.0.zip'), + call( + 'wget -c http://librepo.net/lib/oracle/instantclient-sdk-linux-x86-64-11.2.0.2.0.zip' + ), + call( + 'unzip instantclient-sdk-linux-x86-64-11.2.0.2.0.zip -d oracle_client' + ), + call('rm /home/django/instantclient-sdk-linux-x86-64-11.2.0.2.0.zip'), + call( + 'wget -c http://librepo.net/lib/oracle/instantclient-sqlplus-linux-x86-64-11.2.0.2.0.zip' + ), + call( + 'unzip instantclient-sqlplus-linux-x86-64-11.2.0.2.0.zip -d oracle_client' + ), + call( + 'rm /home/django/instantclient-sqlplus-linux-x86-64-11.2.0.2.0.zip' + ), + call('ln -s libclntsh.so.* libclntsh.so'), + call('pwd'), + call( + "echo '# ORACLE CLIENT CONFIGURATION\nexport ORACLE_HOME=/home/django/oracle_client/instantclient_11_2\nexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ORACLE_HOME' >> .bashrc" + ), + call('source .bashrc'), + call( + 'echo /home/django/oracle_client/instantclient_11_2 > /etc/ld.so.conf.d/oracle.conf' + ), + call('ldconfig'), + ], + ) del env['oracle_client_version'] install_oracle_client() @@ -298,8 +337,7 @@ def test_install_oracle_jdk(self, oracle_jdk_install): install_oracle_jdk(version='7u25-b15') self.assertTrue(oracle_jdk_install.called) - self.assertEqual( - oracle_jdk_install.call_args, call(version='7u25-b15')) + self.assertEqual(oracle_jdk_install.call_args, call(version='7u25-b15')) @patch("fabtools.oracle_jdk.version", return_value=Mock()) def test_oracle_jdk_version(self, jdk_version): @@ -328,8 +366,9 @@ def tearDown(self): @patch("fabric.api.sudo", return_value=Mock()) @patch("fabtools.require.deb.source", return_value=Mock()) @patch("pydiploy.require.system.package_installed", return_value=True) - def test_install_mongodb(self, is_package_installed, deb_source, api_sudo, - uptodate_index, deb_package): + def test_install_mongodb( + self, is_package_installed, deb_source, api_sudo, uptodate_index, deb_package + ): # nothing to do install_mongodb() @@ -341,17 +380,25 @@ def test_install_mongodb(self, is_package_installed, deb_source, api_sudo, self.assertTrue(is_package_installed.called) self.assertTrue(deb_source.called) - self.assertEqual(deb_source.call_args, - call('mongodb', - 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', - 'dist', '10gen')) + self.assertEqual( + deb_source.call_args, + call( + 'mongodb', + 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', + 'dist', + '10gen', + ), + ) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, - call('apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10')) + self.assertEqual( + api_sudo.call_args, + call( + 'apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10' + ), + ) self.assertTrue(uptodate_index.called) self.assertTrue(deb_package.called) - self.assertEqual(deb_package.call_args, - call('mongodb-10gen')) + self.assertEqual(deb_package.call_args, call('mongodb-10gen')) class SAPCheck(TestCase): @@ -383,8 +430,9 @@ def tearDown(self): @patch("fabric.api.cd", return_value=Mock()) @patch("fabtools.require.files.directory", return_value=Mock()) @patch("fabtools.require.deb.packages", return_value=Mock()) - def test_install_sap_client(self, deb_packages, files_directory, - api_cd, api_sudo, files_is_link, api_abort): + def test_install_sap_client( + self, deb_packages, files_directory, api_cd, api_sudo, files_is_link, api_abort + ): api_cd.return_value.__exit__ = Mock() api_cd.return_value.__enter__ = Mock() @@ -392,25 +440,27 @@ def test_install_sap_client(self, deb_packages, files_directory, install_sap_client() self.assertTrue(deb_packages.called) - self.assertEqual(deb_packages.call_args, - call(['libstdc++5'])) + self.assertEqual(deb_packages.call_args, call(['libstdc++5'])) self.assertTrue(files_directory.called) - self.assertEqual(files_directory.call_args, - call(path='/usr/sap', use_sudo=True, mode='755')) + self.assertEqual( + files_directory.call_args, call(path='/usr/sap', use_sudo=True, mode='755') + ) self.assertTrue(api_cd.called) - self.assertEqual(api_cd.call_args_list, - [call('/usr/sap'), call('/lib')]) + self.assertEqual(api_cd.call_args_list, [call('/usr/sap'), call('/lib')]) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args_list, [ - call('wget -c http://librepo.net/lib/sap/rfcsdk_64.tar.gz'), - call('tar xvf rfcsdk_64.tar.gz'), - call('chmod -R 755 rfcsdk'), - call('rm rfcsdk_64.tar.gz'), - call('ln -s /usr/sap/rfcsdk/lib/librfccm.so .') - ]) + self.assertEqual( + api_sudo.call_args_list, + [ + call('wget -c http://librepo.net/lib/sap/rfcsdk_64.tar.gz'), + call('tar xvf rfcsdk_64.tar.gz'), + call('chmod -R 755 rfcsdk'), + call('rm rfcsdk_64.tar.gz'), + call('ln -s /usr/sap/rfcsdk/lib/librfccm.so .'), + ], + ) del env['sap_download_url'] install_sap_client() diff --git a/tests/test_require_django.py b/tests/test_require_django.py index dcf1d61..f59ed94 100644 --- a/tests/test_require_django.py +++ b/tests/test_require_django.py @@ -7,14 +7,21 @@ from unittest import TestCase from fabric.api import env -from mock import call, Mock, patch -from pydiploy.require.django.command import (django_custom_cmd, - django_dump_database, - django_get_version, - django_prepare) -from pydiploy.require.django.utils import (app_settings, deploy_manage_file, - deploy_wsgi_file, extract_settings, - generate_secret_key) +from mock import Mock, call, patch + +from pydiploy.require.django.command import ( + django_custom_cmd, + django_dump_database, + django_get_version, + django_prepare, +) +from pydiploy.require.django.utils import ( + app_settings, + deploy_manage_file, + deploy_wsgi_file, + extract_settings, + generate_secret_key, +) class CommandCheck(TestCase): @@ -31,8 +38,7 @@ def setUp(self): env.remote_owner = "remote_owner" env.remote_base_package_dir = "remote_base_package_dir" env.dest_path = "" - self.dump_name = '%s.json' % datetime.datetime.today().strftime( - "%Y_%m_%d-%H%M") + self.dump_name = '%s.json' % datetime.datetime.today().strftime("%Y_%m_%d-%H%M") def tearDown(self): env.clear() @@ -46,7 +52,17 @@ def tearDown(self): @patch('fabric.api.sudo', return_value=Mock()) @patch('fabric.api.get', return_value=Mock()) @patch('pydiploy.require.django.command.django_get_version', return_value="1.8") - def test_django_prepare(self, django_get_version, api_get, api_sudo, api_settings, api_cd, python_virtualenv, api_local, api_lcd): + def test_django_prepare( + self, + django_get_version, + api_get, + api_sudo, + api_settings, + api_cd, + python_virtualenv, + api_local, + api_lcd, + ): api_lcd.return_value.__exit__ = Mock() api_lcd.return_value.__enter__ = Mock() @@ -63,8 +79,7 @@ def test_django_prepare(self, django_get_version, api_get, api_sudo, api_setting django_prepare() self.assertTrue(python_virtualenv.called) - self.assertEqual( - python_virtualenv.call_args, call('remote_virtualenv_dir')) + self.assertEqual(python_virtualenv.call_args, call('remote_virtualenv_dir')) self.assertTrue(api_cd.called) self.assertEqual(api_cd.call_args, call('remote_current_path')) @@ -76,18 +91,22 @@ def test_django_prepare(self, django_get_version, api_get, api_sudo, api_setting self.assertEqual(api_settings.call_args, call(warn_only=True)) self.assertTrue(api_get.called) - self.assertEqual(api_get.call_args, call( - 'remote_current_path/assets', local_path='local_tmp_dir')) + self.assertEqual( + api_get.call_args, + call('remote_current_path/assets', local_path='local_tmp_dir'), + ) self.assertTrue(api_sudo.called) self.assertEqual( - api_sudo.call_args_list, [ + api_sudo.call_args_list, + [ # call('python -c "import django;print(django.get_version())"'), # call('python manage.py syncdb --noinput'), call('python manage.py migrate'), call('python manage.py compilemessages'), - call('python manage.py collectstatic --noinput')]) - + call('python manage.py collectstatic --noinput'), + ], + ) @patch('fabric.api.lcd', return_value=Mock()) @@ -98,7 +117,17 @@ def test_django_prepare(self, django_get_version, api_get, api_sudo, api_setting @patch('fabric.api.sudo', return_value=Mock()) @patch('fabric.api.get', return_value=Mock()) @patch('pydiploy.require.django.command.django_get_version', return_value="1.6") - def test_django_prepare(self, django_get_version, api_get, api_sudo, api_settings, api_cd, python_virtualenv, api_local, api_lcd): + def test_django_prepare( + self, + django_get_version, + api_get, + api_sudo, + api_settings, + api_cd, + python_virtualenv, + api_local, + api_lcd, + ): api_lcd.return_value.__exit__ = Mock() api_lcd.return_value.__enter__ = Mock() @@ -115,8 +144,7 @@ def test_django_prepare(self, django_get_version, api_get, api_sudo, api_setting django_prepare() self.assertTrue(python_virtualenv.called) - self.assertEqual( - python_virtualenv.call_args, call('remote_virtualenv_dir')) + self.assertEqual(python_virtualenv.call_args, call('remote_virtualenv_dir')) self.assertTrue(api_cd.called) self.assertEqual(api_cd.call_args, call('remote_current_path')) @@ -128,19 +156,22 @@ def test_django_prepare(self, django_get_version, api_get, api_sudo, api_setting self.assertEqual(api_settings.call_args, call(warn_only=True)) self.assertTrue(api_get.called) - self.assertEqual(api_get.call_args, call( - 'remote_current_path/assets', local_path='local_tmp_dir')) + self.assertEqual( + api_get.call_args, + call('remote_current_path/assets', local_path='local_tmp_dir'), + ) self.assertTrue(api_sudo.called) self.assertEqual( - api_sudo.call_args_list, [ + api_sudo.call_args_list, + [ # call('python -c "import django;print(django.get_version())"'), call('python manage.py syncdb --noinput'), call('python manage.py migrate'), call('python manage.py compilemessages'), - call('python manage.py collectstatic --noinput')]) - - + call('python manage.py collectstatic --noinput'), + ], + ) @patch('fabtools.python.virtualenv', return_value=Mock()) @@ -148,7 +179,9 @@ def test_django_prepare(self, django_get_version, api_get, api_sudo, api_setting @patch('fabric.api.settings', return_value=Mock()) @patch('fabric.api.sudo', return_value=Mock()) @patch('fabric.api.get', return_value=Mock()) - def test_django_dump_database(self, api_get, api_sudo, api_settings, api_cd, python_virtualenv): + def test_django_dump_database( + self, api_get, api_sudo, api_settings, api_cd, python_virtualenv + ): python_virtualenv.return_value.__exit__ = Mock() python_virtualenv.return_value.__enter__ = Mock() @@ -162,23 +195,25 @@ def test_django_dump_database(self, api_get, api_sudo, api_settings, api_cd, pyt django_dump_database() self.assertTrue(python_virtualenv.called) - self.assertEqual( - python_virtualenv.call_args, call('remote_virtualenv_dir')) + self.assertEqual(python_virtualenv.call_args, call('remote_virtualenv_dir')) self.assertTrue(api_cd.called) self.assertEqual(api_cd.call_args, call('remote_current_path')) self.assertTrue(api_settings.called) - self.assertEqual( - api_settings.call_args, call(sudo_user=env.remote_owner)) + self.assertEqual(api_settings.call_args, call(sudo_user=env.remote_owner)) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, call( - 'python manage.py dumpdata --indent=4 > /tmp/%s ' % self.dump_name)) + self.assertEqual( + api_sudo.call_args, + call('python manage.py dumpdata --indent=4 > /tmp/%s ' % self.dump_name), + ) self.assertTrue(api_get.called) - self.assertEqual(api_get.call_args, call( - '/tmp/%s' % self.dump_name, local_path=env.dest_path)) + self.assertEqual( + api_get.call_args, + call('/tmp/%s' % self.dump_name, local_path=env.dest_path), + ) @patch('fabtools.python.virtualenv', return_value=Mock()) @patch('fabric.api.cd', return_value=Mock()) @@ -195,13 +230,13 @@ def test_django_custom_cmd(self, api_sudo, api_settings, api_cd, python_virtuale api_settings.return_value.__exit__ = Mock() api_settings.return_value.__enter__ = Mock() django_custom_cmd('test') - - @patch('fabtools.python.virtualenv', return_value=Mock()) @patch('fabric.api.cd', return_value=Mock()) @patch('fabric.api.settings', return_value=Mock()) @patch('fabric.api.sudo', return_value=Mock()) - def test_django_get_version(self, api_sudo, api_settings, api_cd, python_virtualenv): + def test_django_get_version( + self, api_sudo, api_settings, api_cd, python_virtualenv + ): python_virtualenv.return_value.__exit__ = Mock() python_virtualenv.return_value.__enter__ = Mock() @@ -215,8 +250,7 @@ def test_django_get_version(self, api_sudo, api_settings, api_cd, python_virtual django_get_version() self.assertTrue(python_virtualenv.called) - self.assertEqual( - python_virtualenv.call_args, call('remote_virtualenv_dir')) + self.assertEqual(python_virtualenv.call_args, call('remote_virtualenv_dir')) self.assertTrue(api_cd.called) self.assertEqual(api_cd.call_args, call('remote_current_path')) @@ -226,9 +260,9 @@ def test_django_get_version(self, api_sudo, api_settings, api_cd, python_virtual self.assertTrue(api_sudo.called) self.assertEqual( - api_sudo.call_args_list, [ - call('python -c "import django;print(django.get_version())"')]) - + api_sudo.call_args_list, + [call('python -c "import django;print(django.get_version())"')], + ) class UtilsCheck(TestCase): @@ -245,8 +279,11 @@ def setUp(self): env.map_settings = { "secret_key": "SECRET_KEY", "default_db_user": "DATABASES['default']['USER']", - 'thanks_arno': ('THANKS_ARNAUD', - r'environ.get\(["\']THANKS_ARNAUD[\'"], [\'"](.*)["\']\)')} + 'thanks_arno': ( + 'THANKS_ARNAUD', + r'environ.get\(["\']THANKS_ARNAUD[\'"], [\'"](.*)["\']\)', + ), + } env.local_tmp_root_app_package = "local_tmp_root_app_package" env.remote_owner = "owner" env.previous_settings_file = "remote_settings_file" @@ -268,8 +305,9 @@ def test_extract_settings(self, api_get): extract_settings() self.assertTrue(api_get.called) - self.assertEqual(api_get.call_args, call( - 'remote_settings_file', local_path='tests/data')) + self.assertEqual( + api_get.call_args, call('remote_settings_file', local_path='tests/data') + ) self.assertEqual(env.secret_key, '{{ secret_key }}') self.assertEqual(env.default_db_user, '{{ default_db_user }}') @@ -284,12 +322,12 @@ def test_app_settings(self, upload_template, api_require, api_execute, is_file): app_settings(test1='toto') self.assertTrue(is_file.called) - self.assertEqual(is_file.call_args, call( - path='remote_settings_file', use_sudo=True)) + self.assertEqual( + is_file.call_args, call(path='remote_settings_file', use_sudo=True) + ) self.assertTrue(api_execute.called) - self.assertTrue( - str(api_execute.call_args).find('extract_settings') > 0) + self.assertTrue(str(api_execute.call_args).find('extract_settings') > 0) self.assertEqual(env.test1, 'toto') @@ -298,16 +336,18 @@ def test_app_settings(self, upload_template, api_require, api_execute, is_file): self.assertTrue(str(api_require.call_args).find('default_db_user') > 0) self.assertTrue(upload_template.called) - self.assertTrue(str(upload_template.call_args).find( - "template_dir='local_tmp_root_app_package/settings'") > 0) - self.assertTrue( - str(upload_template.call_args).find("'settings.py'") > 0) - self.assertTrue( - str(upload_template.call_args).find("'remote_settings_file'") > 0) self.assertTrue( - str(upload_template.call_args).find("use_jinja=True") > 0) + str(upload_template.call_args).find( + "template_dir='local_tmp_root_app_package/settings'" + ) + > 0 + ) + self.assertTrue(str(upload_template.call_args).find("'settings.py'") > 0) self.assertTrue( - str(upload_template.call_args).find("user='owner'") > 0) + str(upload_template.call_args).find("'remote_settings_file'") > 0 + ) + self.assertTrue(str(upload_template.call_args).find("use_jinja=True") > 0) + self.assertTrue(str(upload_template.call_args).find("user='owner'") > 0) # is file false is_file.return_value = False @@ -315,12 +355,12 @@ def test_app_settings(self, upload_template, api_require, api_execute, is_file): app_settings() self.assertTrue(is_file.called) - self.assertEqual(is_file.call_args, call( - path='remote_settings_file', use_sudo=True)) + self.assertEqual( + is_file.call_args, call(path='remote_settings_file', use_sudo=True) + ) self.assertTrue(api_execute.called) - self.assertTrue( - str(api_execute.call_args).find('generate_secret_key') > 0) + self.assertTrue(str(api_execute.call_args).find('generate_secret_key') > 0) @patch('fabtools.files.upload_template', return_value=Mock()) def test_deploy_manage_file(self, upload_template): diff --git a/tests/test_require_git.py b/tests/test_require_git.py index a9d4ac5..1179701 100644 --- a/tests/test_require_git.py +++ b/tests/test_require_git.py @@ -5,9 +5,13 @@ from unittest import TestCase from fabric.api import env -from mock import call, Mock, patch -from pydiploy.require.git import (archive, check_tag_exist, collect_branches, - collect_tags) +from mock import Mock, call, patch +from pydiploy.require.git import ( + archive, + check_tag_exist, + collect_branches, + collect_tags, +) class GitCheck(TestCase): @@ -41,26 +45,42 @@ def test_archive(self, api_local, api_lcd, api_abort): self.assertEqual(api_lcd.call_args, call('.')) self.assertTrue(api_local.called) - self.assertEqual(api_local.call_args, call( - 'git archive --format=tar HEAD |gzip > /tmp/myfile.tar.gz')) + self.assertEqual( + api_local.call_args, + call('git archive --format=tar HEAD |gzip > /tmp/myfile.tar.gz'), + ) # if remote and prefix archive(self.filename, remote=self.remote, prefix=self.prefix) self.assertTrue(api_local.called) - self.assertEqual(api_local.call_args, call( - 'git archive --format=tar --remote=remote --prefix=prefix HEAD |gzip > /tmp/myfile.tar.gz')) + self.assertEqual( + api_local.call_args, + call( + 'git archive --format=tar --remote=remote --prefix=prefix HEAD |gzip > /tmp/myfile.tar.gz' + ), + ) # if remote and prefix and specific folder - archive(self.filename, remote=self.remote, prefix=self.prefix, specific_folder='/myfolder') + archive( + self.filename, + remote=self.remote, + prefix=self.prefix, + specific_folder='/myfolder', + ) self.assertTrue(api_local.called) - self.assertEqual(api_local.call_args, call( - 'git archive --format=tar --remote=remote --prefix=prefix HEAD:/myfolder |gzip > /tmp/myfile.tar.gz')) + self.assertEqual( + api_local.call_args, + call( + 'git archive --format=tar --remote=remote --prefix=prefix HEAD:/myfolder |gzip > /tmp/myfile.tar.gz' + ), + ) # if format not supported archive(self.filename, format="error") self.assertTrue(api_abort.called) - self.assertEqual(api_abort.call_args, call( - 'Git archive format not supported: error')) + self.assertEqual( + api_abort.call_args, call('Git archive format not supported: error') + ) @patch('fabric.api.lcd', return_value=Mock()) @patch('fabric.context_managers.hide', return_value=Mock()) diff --git a/tests/test_require_nginx.py b/tests/test_require_nginx.py index 542d952..dd8678a 100644 --- a/tests/test_require_nginx.py +++ b/tests/test_require_nginx.py @@ -5,13 +5,22 @@ from unittest import TestCase from fabric.api import env -from mock import call, Mock, patch -from pydiploy.require.nginx import (down_site_config, web_static_files, - nginx_pkg, nginx_reload, nginx_restart, - nginx_start, nginx_started, root_web, - set_website_down, set_website_up, - up_site_config, upload_maintenance_page, - web_configuration) +from mock import Mock, call, patch +from pydiploy.require.nginx import ( + down_site_config, + nginx_pkg, + nginx_reload, + nginx_restart, + nginx_start, + nginx_started, + root_web, + set_website_down, + set_website_up, + up_site_config, + upload_maintenance_page, + web_configuration, + web_static_files, +) class NginxCheck(TestCase): @@ -37,8 +46,16 @@ def tearDown(self): def test_root_web(self, files_directory): root_web() self.assertTrue(files_directory.called) - self.assertEqual(files_directory.call_args, call( - 'remote_static_root', owner='root', use_sudo=True, group='root', mode='755')) + self.assertEqual( + files_directory.call_args, + call( + 'remote_static_root', + owner='root', + use_sudo=True, + group='root', + mode='755', + ), + ) @patch('fabtools.require.deb.packages', return_value=Mock()) def test_nginx_pkg(self, deb_packages): @@ -246,8 +263,16 @@ def test_nginx_started_systemd(self, is_active, is_systemd): def test_web_static_files(self, rsync_project): web_static_files() self.assertTrue(rsync_project.called) - self.assertEqual(rsync_project.call_args, - call('remote_static_root/application_name', 'local_tmp_dir/assets/', extra_opts='--rsync-path="sudo rsync" --exclude="maintenance.html"', delete=True, ssh_opts='-t')) + self.assertEqual( + rsync_project.call_args, + call( + 'remote_static_root/application_name', + 'local_tmp_dir/assets/', + extra_opts='--rsync-path="sudo rsync" --exclude="maintenance.html"', + delete=True, + ssh_opts='-t', + ), + ) @patch('pydiploy.require.nginx.up_site_config', return_value=True) @patch('pydiploy.require.nginx.down_site_config', return_value=True) @@ -255,7 +280,9 @@ def test_web_static_files(self, rsync_project): @patch('fabtools.files.is_link', return_value=True) @patch('fabric.api.cd', return_value=Mock()) @patch('fabric.api.sudo', return_value=Mock()) - def test_web_configuration(self, api_sudo, api_cd, is_link, upload_template, up_site_cfg, down_site_cfg): + def test_web_configuration( + self, api_sudo, api_cd, is_link, upload_template, up_site_cfg, down_site_cfg + ): api_cd.return_value.__exit__ = Mock() api_cd.return_value.__enter__ = Mock() @@ -266,12 +293,10 @@ def test_web_configuration(self, api_sudo, api_cd, is_link, upload_template, up_ web_configuration() self.assertTrue(api_cd.called) - self.assertEqual(api_cd.call_args, - call('/etc/nginx/sites-enabled')) + self.assertEqual(api_cd.call_args, call('/etc/nginx/sites-enabled')) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, - call('rm -f default')) + self.assertEqual(api_sudo.call_args, call('rm -f default')) @patch('fabtools.files.upload_template', return_value=Mock()) @patch('fabtools.files.is_link', return_value=True) @@ -284,12 +309,16 @@ def test_up_site_conf(self, api_sudo, api_cd, is_link, upload_template): up_site_config() self.assertTrue(upload_template.called) + self.assertTrue(str(upload_template.call_args).find("'nginx.conf.tpl'") > 0) self.assertTrue( - str(upload_template.call_args).find("'nginx.conf.tpl'") > 0) - self.assertTrue(str(upload_template.call_args).find( - "'/etc/nginx/sites-available/server_name.conf'") > 0) - self.assertTrue(str(upload_template.call_args) - .find("template_dir='lib_path/templates'") > 0) + str(upload_template.call_args).find( + "'/etc/nginx/sites-available/server_name.conf'" + ) + > 0 + ) + self.assertTrue( + str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0 + ) self.assertTrue(upload_template.is_link) @@ -298,12 +327,13 @@ def test_up_site_conf(self, api_sudo, api_cd, is_link, upload_template): up_site_config() self.assertTrue(api_cd.called) - self.assertEqual(api_cd.call_args, - call('/etc/nginx/sites-enabled')) + self.assertEqual(api_cd.call_args, call('/etc/nginx/sites-enabled')) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, call( - 'ln -s /etc/nginx/sites-available/server_name.conf .')) + self.assertEqual( + api_sudo.call_args, + call('ln -s /etc/nginx/sites-available/server_name.conf .'), + ) @patch('pydiploy.require.nginx.upload_maintenance_page') @patch('fabtools.files.upload_template', return_value=Mock()) @@ -314,11 +344,17 @@ def test_down_site_conf(self, upload_template, maintenance_page): self.assertTrue(upload_template.called) self.assertTrue(maintenance_page.called) self.assertTrue( - str(upload_template.call_args).find("'nginx_down.conf.tpl'") > 0) - self.assertTrue(str(upload_template.call_args).find( - "'/etc/nginx/sites-available/server_name_down.conf'") > 0) - self.assertTrue(str(upload_template.call_args) - .find("template_dir='lib_path/templates'") > 0) + str(upload_template.call_args).find("'nginx_down.conf.tpl'") > 0 + ) + self.assertTrue( + str(upload_template.call_args).find( + "'/etc/nginx/sites-available/server_name_down.conf'" + ) + > 0 + ) + self.assertTrue( + str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0 + ) @patch('fabtools.files.upload_template', return_value=Mock()) @patch('fabtools.files.is_link', return_value=True) @@ -332,23 +368,28 @@ def test_up_site_conf(self, api_sudo, api_cd, is_link, upload_template): up_site_config() self.assertTrue(upload_template.called) + self.assertTrue(str(upload_template.call_args).find("'nginx.conf.tpl'") > 0) self.assertTrue( - str(upload_template.call_args).find("'nginx.conf.tpl'") > 0) - self.assertTrue(str(upload_template.call_args).find( - "'/etc/nginx/sites-available/server_name.conf'") > 0) - self.assertTrue(str(upload_template.call_args) - .find("template_dir='lib_path/templates'") > 0) + str(upload_template.call_args).find( + "'/etc/nginx/sites-available/server_name.conf'" + ) + > 0 + ) + self.assertTrue( + str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0 + ) is_link.return_value = False up_site_config() self.assertTrue(api_cd.called) - self.assertEqual(api_cd.call_args, - call('/etc/nginx/sites-enabled')) + self.assertEqual(api_cd.call_args, call('/etc/nginx/sites-enabled')) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, call( - 'ln -s /etc/nginx/sites-available/server_name.conf .')) + self.assertEqual( + api_sudo.call_args, + call('ln -s /etc/nginx/sites-available/server_name.conf .'), + ) @patch('pydiploy.require.nginx.nginx_reload', return_value=Mock()) @patch('fabric.api.sudo', return_value=Mock()) @@ -356,18 +397,18 @@ def test_up_site_conf(self, api_sudo, api_cd, is_link, upload_template): @patch('fabtools.files.is_link', return_value=True) @patch('pydiploy.require.nginx.up_site_config', return_value=Mock()) @patch('fabtools.files.is_file', return_value=True) - def test_set_website_up(self, is_file, up_site_config, is_link, api_cd, api_sudo, nginx_reload): + def test_set_website_up( + self, is_file, up_site_config, is_link, api_cd, api_sudo, nginx_reload + ): api_cd.return_value.__exit__ = Mock() api_cd.return_value.__enter__ = Mock() set_website_up() self.assertTrue(api_cd.called) - self.assertEqual(api_cd.call_args, - call('/etc/nginx/sites-enabled')) + self.assertEqual(api_cd.call_args, call('/etc/nginx/sites-enabled')) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, call( - 'rm -f server_name_down.conf')) + self.assertEqual(api_sudo.call_args, call('rm -f server_name_down.conf')) # no up config file is_file.return_value = False @@ -380,12 +421,13 @@ def test_set_website_up(self, is_file, up_site_config, is_link, api_cd, api_sudo set_website_up() self.assertTrue(api_cd.called) - self.assertEqual(api_cd.call_args, - call('/etc/nginx/sites-enabled')) + self.assertEqual(api_cd.call_args, call('/etc/nginx/sites-enabled')) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, call( - 'ln -s /etc/nginx/sites-available/server_name.conf .')) + self.assertEqual( + api_sudo.call_args, + call('ln -s /etc/nginx/sites-available/server_name.conf .'), + ) @patch('pydiploy.require.nginx.nginx_reload', return_value=Mock()) @patch('fabric.api.sudo', return_value=Mock()) @@ -393,18 +435,18 @@ def test_set_website_up(self, is_file, up_site_config, is_link, api_cd, api_sudo @patch('fabtools.files.is_link', return_value=True) @patch('pydiploy.require.nginx.down_site_config', return_value=Mock()) @patch('fabtools.files.is_file', return_value=True) - def test_set_website_down(self, is_file, down_site_config, is_link, api_cd, api_sudo, nginx_reload): + def test_set_website_down( + self, is_file, down_site_config, is_link, api_cd, api_sudo, nginx_reload + ): api_cd.return_value.__exit__ = Mock() api_cd.return_value.__enter__ = Mock() set_website_down() self.assertTrue(api_cd.called) - self.assertEqual(api_cd.call_args, - call('/etc/nginx/sites-enabled')) + self.assertEqual(api_cd.call_args, call('/etc/nginx/sites-enabled')) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, call( - 'rm -f server_name.conf')) + self.assertEqual(api_sudo.call_args, call('rm -f server_name.conf')) # no up config file is_file.return_value = False @@ -417,24 +459,29 @@ def test_set_website_down(self, is_file, down_site_config, is_link, api_cd, api_ set_website_down() self.assertTrue(api_cd.called) - self.assertEqual(api_cd.call_args, - call('/etc/nginx/sites-enabled')) + self.assertEqual(api_cd.call_args, call('/etc/nginx/sites-enabled')) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, call( - 'ln -s /etc/nginx/sites-available/server_name_down.conf .')) + self.assertEqual( + api_sudo.call_args, + call('ln -s /etc/nginx/sites-available/server_name_down.conf .'), + ) @patch('fabtools.files.upload_template', return_value=Mock()) def test_upload_maintenance_page(self, upload_template): upload_maintenance_page() self.assertTrue(upload_template.called) - self.assertTrue(str(upload_template.call_args) - .find("maintenance.html.tpl") > 0) - self.assertTrue(str(upload_template.call_args) - .find("remote_static_root/application_name/maintenance.html") > 0) - self.assertTrue(str(upload_template.call_args) - .find("template_dir='lib_path/templates'") > 0) + self.assertTrue(str(upload_template.call_args).find("maintenance.html.tpl") > 0) + self.assertTrue( + str(upload_template.call_args).find( + "remote_static_root/application_name/maintenance.html" + ) + > 0 + ) + self.assertTrue( + str(upload_template.call_args).find("template_dir='lib_path/templates'") > 0 + ) env.maintenance_text = "ééééééééééééééééééé" env.maintenance_title = "ààààààà" diff --git a/tests/test_require_python.py b/tests/test_require_python.py index ba0161b..04d4c42 100644 --- a/tests/test_require_python.py +++ b/tests/test_require_python.py @@ -36,16 +36,22 @@ def tearDown(self): def test_python_pkg(self, dist_id, dist_rls, python_install, deb_packages): python_pkg() self.assertTrue(deb_packages.called) - self.assertEqual(deb_packages.call_args, call(['python-dev', 'python-pip'], update=False)) + self.assertEqual( + deb_packages.call_args, call(['python-dev', 'python-pip'], update=False) + ) self.assertTrue(python_install.called) - self.assertEqual(python_install.call_args, call('pip', upgrade=True, use_sudo=True)) + self.assertEqual( + python_install.call_args, call('pip', upgrade=True, use_sudo=True) + ) @patch('fabtools.python.virtualenv', return_value=Mock()) @patch('fabric.api.cd', return_value=Mock()) @patch('fabtools.require.python.install', return_value=Mock()) @patch('fabric.api.sudo', return_value=Mock()) @patch('fabtools.python.install_requirements', return_value=Mock()) - def test_application_dependencies(self, install_requirements, api_sudo, python_install, api_cd, python_virtualenv): + def test_application_dependencies( + self, install_requirements, api_sudo, python_install, api_cd, python_virtualenv + ): python_virtualenv.return_value.__exit__ = Mock() python_virtualenv.return_value.__enter__ = Mock() @@ -108,5 +114,10 @@ def test_virtualenv(self, python_virtualenv, files_directory): self.assertTrue(files_directory.called) self.assertEqual( files_directory.call_args, - call('remote_virtualenv_dir', owner='remote_owner', use_sudo=True, group='remote_group'), + call( + 'remote_virtualenv_dir', + owner='remote_owner', + use_sudo=True, + group='remote_group', + ), ) diff --git a/tests/test_require_simple.py b/tests/test_require_simple.py index 24eef09..973a72a 100644 --- a/tests/test_require_simple.py +++ b/tests/test_require_simple.py @@ -4,10 +4,14 @@ import copy import datetime from unittest import TestCase + from fabric.api import env -from mock import call, Mock, patch -from pydiploy.require.simple.utils import (app_settings, deploy_environ_file, - extract_settings) +from mock import Mock, call, patch +from pydiploy.require.simple.utils import ( + app_settings, + deploy_environ_file, + extract_settings, +) class UtilsCheck(TestCase): @@ -24,8 +28,11 @@ def setUp(self): env.map_settings = { "server_url": "SERVER_URL", "default_db_user": "DATABASES['default']['USER']", - 'thanks_arno': ('THANKS_ARNAUD', - r'environ.get\(["\']THANKS_ARNAUD[\'"], [\'"](.*)["\']\)')} + 'thanks_arno': ( + 'THANKS_ARNAUD', + r'environ.get\(["\']THANKS_ARNAUD[\'"], [\'"](.*)["\']\)', + ), + } env.local_tmp_root_app_package = "local_tmp_root_app_package" env.remote_owner = "owner" env.previous_settings_file = "remote_settings_file" @@ -43,8 +50,9 @@ def test_extract_settings(self, api_get): extract_settings() self.assertTrue(api_get.called) - self.assertEqual(api_get.call_args, call( - 'remote_settings_file', local_path='tests/data')) + self.assertEqual( + api_get.call_args, call('remote_settings_file', local_path='tests/data') + ) self.assertEqual(env.server_url, '{{ server_url }}') self.assertEqual(env.default_db_user, '{{ default_db_user }}') @@ -59,12 +67,12 @@ def test_app_settings(self, upload_template, api_require, api_execute, is_file): app_settings(test1='toto') self.assertTrue(is_file.called) - self.assertEqual(is_file.call_args, call( - path='remote_settings_file', use_sudo=True)) + self.assertEqual( + is_file.call_args, call(path='remote_settings_file', use_sudo=True) + ) self.assertTrue(api_execute.called) - self.assertTrue( - str(api_execute.call_args).find('extract_settings') > 0) + self.assertTrue(str(api_execute.call_args).find('extract_settings') > 0) self.assertEqual(env.test1, 'toto') @@ -73,16 +81,18 @@ def test_app_settings(self, upload_template, api_require, api_execute, is_file): self.assertTrue(str(api_require.call_args).find('default_db_user') > 0) self.assertTrue(upload_template.called) - self.assertTrue(str(upload_template.call_args).find( - "template_dir='local_tmp_root_app_package/settings'") > 0) - self.assertTrue( - str(upload_template.call_args).find("'settings.py'") > 0) - self.assertTrue( - str(upload_template.call_args).find("'remote_settings_file'") > 0) self.assertTrue( - str(upload_template.call_args).find("use_jinja=True") > 0) + str(upload_template.call_args).find( + "template_dir='local_tmp_root_app_package/settings'" + ) + > 0 + ) + self.assertTrue(str(upload_template.call_args).find("'settings.py'") > 0) self.assertTrue( - str(upload_template.call_args).find("user='owner'") > 0) + str(upload_template.call_args).find("'remote_settings_file'") > 0 + ) + self.assertTrue(str(upload_template.call_args).find("use_jinja=True") > 0) + self.assertTrue(str(upload_template.call_args).find("user='owner'") > 0) # is file false is_file.return_value = False @@ -90,8 +100,9 @@ def test_app_settings(self, upload_template, api_require, api_execute, is_file): app_settings() self.assertTrue(is_file.called) - self.assertEqual(is_file.call_args, call( - path='remote_settings_file', use_sudo=True)) + self.assertEqual( + is_file.call_args, call(path='remote_settings_file', use_sudo=True) + ) @patch('fabtools.files.upload_template', return_value=Mock()) def test_deploy_environ_file(self, upload_template): diff --git a/tests/test_require_system.py b/tests/test_require_system.py index 8dbbb07..1929c9c 100644 --- a/tests/test_require_system.py +++ b/tests/test_require_system.py @@ -49,13 +49,24 @@ def test_add_user(self, require_group, require_user, require_sudoer): self.assertTrue(require_group.called) self.assertEqual( require_group.call_args, - call('%remote_group', passwd=False, commands='mycommand', operators='remote_owner,root'), + call( + '%remote_group', + passwd=False, + commands='mycommand', + operators='remote_owner,root', + ), ) self.assertTrue(require_user.called) self.assertEqual( require_user.call_args, - call('remote_owner', create_group=True, shell='/bin/bash', group='remote_group', create_home=True), + call( + 'remote_owner', + create_group=True, + shell='/bin/bash', + group='remote_group', + create_home=True, + ), ) self.assertTrue(require_sudoer.called) @@ -88,7 +99,8 @@ def test_set_locale(self, api_sudo, api_run): self.assertTrue(api_run.called) self.assertTrue(api_sudo.called) self.assertEqual( - api_sudo.call_args_list, [call('locale-gen FR_fr'), call('/usr/sbin/update-locale LANG=FR_fr')] + api_sudo.call_args_list, + [call('locale-gen FR_fr'), call('/usr/sbin/update-locale LANG=FR_fr')], ) @patch('fabtools.files.is_link', return_value=False) @@ -106,7 +118,10 @@ def test_set_timezone(self, api_sudo, distrib_id, is_link): self.assertTrue(distrib_id.called) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, call('cp -f /usr/share/zoneinfo/mytimezone /etc/localtime')) + self.assertEqual( + api_sudo.call_args, + call('cp -f /usr/share/zoneinfo/mytimezone /etc/localtime'), + ) @patch('fabtools.files.is_link', return_value=True) @patch('fabtools.system.distrib_id', return_value='Notsupported') @@ -123,7 +138,10 @@ def test_set_timezone_link(self, api_sudo, distrib_id, is_link): self.assertTrue(distrib_id.called) self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, call('ln -sf /usr/share/zoneinfo/mytimezone /etc/localtime')) + self.assertEqual( + api_sudo.call_args, + call('ln -sf /usr/share/zoneinfo/mytimezone /etc/localtime'), + ) @patch('fabric.api.sudo', return_value=Mock()) def test_permissions(self, api_sudo): @@ -140,7 +158,9 @@ def test_package_installed(self, api_sudo, api_settings): package_installed("package") self.assertTrue(api_sudo.called) - self.assertEqual(api_sudo.call_args, call('dpkg-query -l "package" | grep -q ^.i')) + self.assertEqual( + api_sudo.call_args, call('dpkg-query -l "package" | grep -q ^.i') + ) @patch('fabtools.require.deb.packages', return_value=Mock()) def test_install_extra_packages(self, deb_pkgs): @@ -162,12 +182,24 @@ def test_install_extra_ppa(self, deb_ppa): def test_install_extra_source(self, deb_source): install_extra_source( - [['mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'],] + [ + [ + 'mongodb', + 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', + 'dist', + '10gen', + ], + ] ) self.assertTrue(deb_source.called) self.assertEqual( deb_source.call_args, - call('mongodb', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', 'dist', '10gen'), + call( + 'mongodb', + 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart', + 'dist', + '10gen', + ), ) @patch('fabtools.require.deb.package', return_value=Mock()) @@ -179,7 +211,15 @@ def test_install_extra_source(self, deb_source): @patch('fabtools.system.distrib_id', return_value='Ubuntu') @patch('pydiploy.require.system.package_installed', return_value=False) def test_check_python3_install( - self, pkg_installed, sys_distrib, sys_distrib_id, deb_pkgs, fab_sudo, deb_ppa, deb_source, deb_pkg + self, + pkg_installed, + sys_distrib, + sys_distrib_id, + deb_pkgs, + fab_sudo, + deb_ppa, + deb_source, + deb_pkg, ): check_python3_install() diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 2d1b5e5..8902150 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -5,7 +5,7 @@ from unittest import TestCase from fabric.api import env -from mock import call, Mock, patch +from mock import Mock, call, patch from pydiploy.scripts import get_dest_files, sublime_text_snippet @@ -18,18 +18,15 @@ class ScriptsCheck(TestCase): def setUp(self): self.previous_env = copy.deepcopy(env) - def tearDown(self): env.clear() env.update(self.previous_env) - def test_get_dest_files(self): get_dest_files('Linux') # test NotImplemented system self.assertRaises(NotImplementedError, get_dest_files, 'amigaos') - def test_sublime_text_snippet(self): sublime_text_snippet() diff --git a/tests/test_simple.py b/tests/test_simple.py index ead0627..93d103c 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -5,14 +5,18 @@ from unittest import TestCase from fabric.api import env -from mock import call, Mock, patch -from pydiploy.simple import (wrap_deploy, application_packages, - deploy_backend, post_install_backend, - pre_install_backend, rollback) +from mock import Mock, call, patch +from pydiploy.simple import ( + application_packages, + deploy_backend, + post_install_backend, + pre_install_backend, + rollback, + wrap_deploy, +) class ReleasesManagerCheck(TestCase): - def setUp(self): self.previous_env = copy.deepcopy(env) env.remote_python_version = 2.7 @@ -32,77 +36,116 @@ def test_wrap_deploy(self, api_execute, api_abort): def test_application_packages(self, api_execute, deb_packages): application_packages() self.assertTrue(deb_packages.called) - self.assertEqual( - deb_packages.call_args, call(['gettext'], update=False)) + self.assertEqual(deb_packages.call_args, call(['gettext'], update=False)) self.assertTrue(api_execute.called) self.assertTrue( - str(api_execute.call_args_list[0]).find('call(= 3 env.remote_python_version = 3 application_packages() self.assertTrue(api_execute.called) self.assertTrue( - str(api_execute.call_args_list[1]).find('call( -Pydiploy fabfile 1.1.7.8 +Pydiploy fabfile 1.1.7.9 pydiployfab source.python \ No newline at end of file diff --git a/tools/pydiployfabfilebottle.sublime-snippet b/tools/pydiployfabfilebottle.sublime-snippet index 68993de..d53262e 100644 --- a/tools/pydiployfabfilebottle.sublime-snippet +++ b/tools/pydiployfabfilebottle.sublime-snippet @@ -6,28 +6,28 @@ Fabfile example file to deploy a standard bottle webapp """ -from fabric.api import (env, roles, execute, task) from os.path import join -from pydiploy.prepare import (tag as pydiploy_tag, - build_env as pydiploy_build_env) -from pydiploy.bottle import (deploy_backend as pydiploy_deploy_backend, - deploy_frontend as pydiploy_deploy_frontend, - rollback as pydiploy_rollback, - post_install_backend as pydiploy_postinstall_backend, - post_install_frontend as pydiploy_postinstall_frontend, - pre_install_backend as pydiploy_preinstall_backend, - pre_install_frontend as pydiploy_preinstall_frontend, - reload_frontend as pydiploy_reload_frontend, - reload_backend as pydiploy_reload_backend, - set_app_up as pydiploy_set_up, - set_app_down as pydiploy_set_down, - install_oracle_client as pydiploy_setup_oracle, - install_postgres_server as pydiploy_setup_postgres) +from fabric.api import env, execute, roles, task +from pydiploy.bottle import deploy_backend as pydiploy_deploy_backend +from pydiploy.bottle import deploy_frontend as pydiploy_deploy_frontend +from pydiploy.bottle import install_oracle_client as pydiploy_setup_oracle +from pydiploy.bottle import install_postgres_server as pydiploy_setup_postgres +from pydiploy.bottle import post_install_backend as pydiploy_postinstall_backend +from pydiploy.bottle import post_install_frontend as pydiploy_postinstall_frontend +from pydiploy.bottle import pre_install_backend as pydiploy_preinstall_backend +from pydiploy.bottle import pre_install_frontend as pydiploy_preinstall_frontend +from pydiploy.bottle import reload_backend as pydiploy_reload_backend +from pydiploy.bottle import reload_frontend as pydiploy_reload_frontend +from pydiploy.bottle import rollback as pydiploy_rollback +from pydiploy.bottle import set_app_down as pydiploy_set_down +from pydiploy.bottle import set_app_up as pydiploy_set_up +from pydiploy.prepare import build_env as pydiploy_build_env +from pydiploy.prepare import tag as pydiploy_tag # edit config here ! -env.use_sudo = True # use sudo or not +env.use_sudo = True # use sudo or not env.remote_owner = 'myremoteuser' env.remote_group = 'myremotegroup' @@ -38,8 +38,7 @@ env.root_package_name = 'myapp' env.remote_home = '/home/myremoteuser' # remote home root env.remote_python_version = 3.4 # python version env.remote_virtualenv_root = join(env.remote_home, '.virtualenvs') # venv root -env.remote_virtualenv_dir = join(env.remote_virtualenv_root, - env.application_name) # venv for webapp dir +env.remote_virtualenv_dir = join(env.remote_virtualenv_root, env.application_name) # venv for webapp dir env.remote_repo_url = 'git@git.net:myapp.git' env.remote_static_root = '/var/www/static' # root of static files @@ -87,7 +86,7 @@ env.req_pydiploy_version = '1.1.6.0' # env.socket_host='localhost' # use it in env method to force a socket host -# env.run_tests_command = 'tox' +#  env.run_tests_command = 'tox' # env.media_folder = '/media' # path of the application's media files # env.remote_media_folder = '/srv/media/myapp' # remote folder of the application's media files @@ -98,6 +97,8 @@ env.req_pydiploy_version = '1.1.6.0' # env.default_db_user = 'myapp_db_user' # env.default_db_password = 'S3CR3T' +# env.no_tag_check = True # When your fabfile is not in your project git repository be sure to set this var + @task def test(): @@ -152,6 +153,7 @@ def prod(): } execute(build_env) + # dont touch after that point if you don't know what you are doing ! @@ -202,7 +204,7 @@ def deploy(): execute(deploy_backend) execute(deploy_frontend) # uncomment this to toggle app to up mode again : - #execute(set_up) + # execute(set_up) @roles('web') @@ -246,6 +248,7 @@ def post_install_frontend(): """Post installation of frontend""" execute(pydiploy_postinstall_frontend) + @roles('web') @task def install_oracle(): @@ -293,7 +296,7 @@ def set_up(): execute(pydiploy_set_up) ]]> -Pydiploy fabfile bottle 1.1.7.8 +Pydiploy fabfile bottle 1.1.7.9 pydiployfabbottle source.python \ No newline at end of file diff --git a/tools/pydiployfabfilesimple.sublime-snippet b/tools/pydiployfabfilesimple.sublime-snippet index a3f30bf..537218b 100644 --- a/tools/pydiployfabfilesimple.sublime-snippet +++ b/tools/pydiployfabfilesimple.sublime-snippet @@ -6,19 +6,19 @@ Fabfile example file to deploy a standard simple webapp """ -from fabric.api import (env, roles, execute, task) from os.path import join -from pydiploy.prepare import (tag as pydiploy_tag, - build_env as pydiploy_build_env) -from pydiploy.simple import (deploy_backend as pydiploy_deploy_backend, - rollback as pydiploy_rollback, - post_install_backend as pydiploy_postinstall_backend, - pre_install_backend as pydiploy_preinstall_backend) +from fabric.api import env, execute, roles, task +from pydiploy.prepare import build_env as pydiploy_build_env +from pydiploy.prepare import tag as pydiploy_tag +from pydiploy.simple import deploy_backend as pydiploy_deploy_backend +from pydiploy.simple import post_install_backend as pydiploy_postinstall_backend +from pydiploy.simple import pre_install_backend as pydiploy_preinstall_backend +from pydiploy.simple import rollback as pydiploy_rollback # edit config here ! -env.use_sudo = True # use sudo or not +env.use_sudo = True # use sudo or not env.remote_owner = 'myremoteuser' env.remote_group = 'myremotegroup' @@ -29,8 +29,7 @@ env.root_package_name = 'myapp' env.remote_home = '/home/myremoteuser' # remote home root env.remote_python_version = 3.4 # python version env.remote_virtualenv_root = join(env.remote_home, '.virtualenvs') # venv root -env.remote_virtualenv_dir = join(env.remote_virtualenv_root, - env.application_name) # venv for webapp dir +env.remote_virtualenv_dir = join(env.remote_virtualenv_root, env.application_name) # venv for webapp dir env.remote_repo_url = 'git@git.net:myapp.git' env.local_tmp_dir = '/tmp' # tmp dir @@ -55,7 +54,7 @@ env.application_type = 'simple' # specify another type of application # env.verbose = True # verbose display for pydiploy default value = True env.req_pydiploy_version = '1.1.5.0' # required pydiploy version for this fabfile -# env.run_tests_command = 'tox' +#  env.run_tests_command = 'tox' # fill and uncomment not to pass parameters in term (eg: fab tag:master test --set default_db_host='localhost',default_db_name='my_app_db' ) # env.default_db_host = 'localhost' @@ -63,14 +62,14 @@ env.req_pydiploy_version = '1.1.5.0' # required pydiploy version for this fabfi # env.default_db_user = 'myapp_db_user' # env.default_db_password = 'S3CR3T' +# env.no_tag_check = True # When your fabfile is not in your project git repository be sure to set this var + @task def test(): """Define test stage""" env.user = 'vagrant' - env.roledefs = { - 'web': ['192.168.1.4'] - } + env.roledefs = {'web': ['192.168.1.4']} env.goal = "test" # env.server_name = "myapp.net" # optional: if you want to use an url for the name of the remote app folder instead of the application name (manual bottle or flask app) env.map_settings = { @@ -80,12 +79,11 @@ def test(): } execute(build_env) + @task def prod(): """Define prod stage""" - env.roledefs = { - 'web': ['myserver.net'] - } + env.roledefs = {'web': ['myserver.net']} env.goal = "prod" # env.server_name = "myapp.net" # optional: if you want to use an url for the name of the remote app folder instead of the application name (manual bottle or flask app) env.map_settings = { @@ -95,6 +93,7 @@ def prod(): } execute(build_env) + # dont touch after that point if you don't know what you are doing ! @@ -162,7 +161,7 @@ def post_install_backend(): execute(pydiploy_postinstall_backend) ]]> -Pydiploy fabfile simple 1.1.7.8 +Pydiploy fabfile simple 1.1.7.9 pydiployfabsimple source.python \ No newline at end of file From e950b5ce793b1f330f384ec7685e8629cbb9c612 Mon Sep 17 00:00:00 2001 From: Thomas Baguet Date: Fri, 5 Jun 2020 11:21:04 +0200 Subject: [PATCH 44/44] Add commit id in json info file --- pydiploy/require/releases_manager.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pydiploy/require/releases_manager.py b/pydiploy/require/releases_manager.py index db9b6ac..749911a 100644 --- a/pydiploy/require/releases_manager.py +++ b/pydiploy/require/releases_manager.py @@ -6,8 +6,9 @@ import fabric import fabtools +from fabric.api import env, hide, warn_only + import pydiploy -from fabric.api import env, warn_only from pydiploy.decorators import do_verbose @@ -116,6 +117,12 @@ def deploy_code(): # remove existing extracted dir from tarball if os.path.exists('%s/%s' % (env.local_tmp_dir, archive_prefix)): fabric.api.local('rm -rf %s' % archive_prefix) + # use tarball to get commit id + with warn_only(): + env.commit_id = fabric.api.local( + 'zcat %s | git get-tar-commit-id' % os.path.basename(tarball), + capture=True, + ) fabric.api.local('tar xvf %s' % os.path.basename(tarball)) # add deployed.json file with open( @@ -124,12 +131,13 @@ def deploy_code(): data = {} data['info'] = [ { - 'tag': env.tag, + 'app_host': env.host_string, + 'repo_url': env.remote_repo_url, 'python_version': env.remote_python_version, 'local_user': env.local_user, - 'app_host': env.host_string, 'socket_port': env.socket_port, - 'repo_url': env.remote_repo_url, + 'tag': env.tag, + 'commit_id': env.commit_id, } ]