[WIP] Using Ansible to install dependencies for bench setup

.travis.yml

@@ -0,0 +1,20 @@
+language: python
+dist: trusty
+sudo: required
+
+python:
+  - "2.7"
+
+install:
+  - sudo apt-get purge -y mysql-common mysql-server mysql-client
+  - sudo python $TRAVIS_BUILD_DIR/installer/install.py --user travis --skip-bench-setup
+  # - sudo bash $TRAVIS_BUILD_DIR/install_scripts/setup_frappe.sh --skip-install-bench --mysql-root-password travis
+  - mkdir -p ~/bench-repo
+  - cp -r $TRAVIS_BUILD_DIR/* ~/bench-repo/
+  - cd ~ && sudo python bench-repo/installer/install.py --only-dependencies
+
+script:
+  - cd ~
+  - sudo pip install --upgrade pip
+  - sudo pip install -e bench-repo
+  - sudo python -m unittest bench.tests.test_setup_production.TestSetupProduction.test_setup_production

README.md

@@ -3,7 +3,7 @@ Bench
 
 The bench allows you to setup Frappe / ERPNext apps on your local Linux (CentOS 6, Debian 7 or Ubuntu) machine or a production server. You can use the bench to serve multiple frappe sites.
 
-To do this install, you must have basic information on how Linux works and should be able to use the command-line. If you are looking easier ways to get started and evaluate ERPNext, [download the Virtual Machine or take a free trial at FrappeCloud.com](https://erpnext.com/use).
+To do this install, you must have basic information on how Linux works and should be able to use the command-line. If you are looking easier ways to get started and evaluate ERPNext, [download the Virtual Machine or take a free trial on ERPNext.com](https://erpnext.com/pricing).
 
 For questions, please join the [developer forum](https://discuss.frappe.io/).
 

bench/app.py

@@ -8,7 +8,7 @@
 import re
 import subprocess
 
-
+logging.basicConfig(level="DEBUG")
 logger = logging.getLogger(__name__)
 
 class MajorVersionUpgradeException(Exception):
@@ -42,7 +42,7 @@ def write_appstxt(apps, bench='.'):
 
 def get_app(app, git_url, branch=None, bench='.', build_asset_files=True, verbose=False):
 	logger.info('getting app {}'.format(app))
-	shallow_clone = '--depth 1' if check_git_for_shallow_clone() and get_config().get('shallow_clone') else ''
+	shallow_clone = '--depth 1' if check_git_for_shallow_clone() and get_config(bench).get('shallow_clone') else ''
 	branch = '--branch {branch}'.format(branch=branch) if branch else ''
 	exec_cmd("git clone {git_url} {branch} {shallow_clone} --origin upstream {app}".format(
 				git_url=git_url,
@@ -54,11 +54,12 @@ def get_app(app, git_url, branch=None, bench='.', build_asset_files=True, verbos
 	install_app(app, bench=bench, verbose=verbose)
 	if build_asset_files:
 		build_assets(bench=bench)
-	conf = get_config()
+	conf = get_config(bench=bench)
 	if conf.get('restart_supervisor_on_update'):
 		restart_supervisor_processes(bench=bench)
 
 def new_app(app, bench='.'):
+	app = app.lower().replace(" ", "_")
 	logger.info('creating new app {}'.format(app))
 	apps = os.path.abspath(os.path.join(bench, 'apps'))
 	if FRAPPE_VERSION == 4:
@@ -70,7 +71,6 @@ def new_app(app, bench='.'):
 
 def install_app(app, bench='.', verbose=False):
 	logger.info('installing {}'.format(app))
-	conf = get_config()
 	# find_links = '--find-links={}'.format(conf.get('wheel_cache_dir')) if conf.get('wheel_cache_dir') else ''
 	find_links = ''
 	exec_cmd("{pip} install {quiet} {find_links} -e {app}".format(
@@ -81,7 +81,7 @@ def install_app(app, bench='.', verbose=False):
 	add_to_appstxt(app, bench=bench)
 
 def pull_all_apps(bench='.'):
-	rebase = '--rebase' if get_config().get('rebase_on_pull') else ''
+	rebase = '--rebase' if get_config(bench).get('rebase_on_pull') else ''
 
 	for app in get_apps(bench=bench):
 		app_dir = get_repo_dir(app, bench=bench)

bench/cli.py

@@ -13,12 +13,13 @@
 from .utils import (build_assets, patch_sites, exec_cmd, update_bench, get_env_cmd, get_frappe, setup_logging,
 					get_config, update_config, restart_supervisor_processes, put_config, default_config, update_requirements,
 					backup_all_sites, backup_site, get_sites, prime_wheel_cache, is_root, set_mariadb_host, drop_privileges,
-					fix_file_perms, fix_prod_setup_perms, set_ssl_certificate, set_ssl_certificate_key, get_cmd_output, post_upgrade,
+					fix_file_perms, fix_prod_setup_perms, set_ssl_certificate, set_ssl_certificate_key,
+					get_cmd_output, post_upgrade, get_bench_name,
 					pre_upgrade, validate_upgrade, PatchError, download_translations_p, setup_socketio, before_update)
 from .app import get_app as _get_app
 from .app import new_app as _new_app
 from .app import pull_all_apps, get_apps, get_current_frappe_version, is_version_upgrade, switch_to_v4, switch_to_v5, switch_to_master, switch_to_develop
-from .config import generate_nginx_config, generate_supervisor_config, generate_redis_cache_config, generate_redis_async_broker_config
+from .config import generate_nginx_config, generate_supervisor_config, generate_redis_cache_config, generate_redis_async_broker_config, generate_redis_celery_broker_config
 from .production_setup import setup_production as _setup_production
 from .migrate_to_v5 import migrate_to_v5
 import os
@@ -74,7 +75,7 @@ def check_uid():
 
 def change_uid():
 	if is_root() and not cmd_requires_root():
-		frappe_user = get_config().get('frappe_user')
+		frappe_user = get_config(".").get('frappe_user')
 		if frappe_user:
 			drop_privileges(uid_name=frappe_user, gid_name=frappe_user)
 			os.environ['HOME'] = pwd.getpwnam(frappe_user).pw_dir
@@ -206,7 +207,7 @@ def _update(pull=False, patch=False, build=False, bench=False, auto=False, resta
 	if not (pull or patch or build or bench or requirements):
 		pull, patch, build, bench, requirements = True, True, True, True, True
 
-	conf = get_config()
+	conf = get_config(".")
 
 	version_upgrade = is_version_upgrade()
 
@@ -407,7 +408,7 @@ def _prime_wheel_cache():
 	prime_wheel_cache(bench='.')
 
 @click.command('release')
-@click.argument('app', type=click.Choice(['frappe', 'erpnext', 'shopping_cart']))
+@click.argument('app', type=click.Choice(['frappe', 'erpnext', 'erpnext_shopify', 'paypal_integration']))
 @click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch']))
 @click.option('--develop', default='develop')
 @click.option('--master', default='master')
@@ -449,6 +450,11 @@ def setup_redis_async_broker():
 	"generate config for redis async broker"
 	generate_redis_async_broker_config()
 
+@click.command('redis-celery-broker')
+def setup_redis_celery_broker():
+	"generate config for redis celery broker"
+	generate_redis_celery_broker_config()
+
 @click.command('production')
 @click.argument('user')
 def setup_production(user):
@@ -496,6 +502,7 @@ def setup_config():
 setup.add_command(setup_supervisor)
 setup.add_command(setup_redis_cache)
 setup.add_command(setup_redis_async_broker)
+setup.add_command(setup_redis_celery_broker)
 setup.add_command(setup_auto_update)
 setup.add_command(setup_dnsmasq)
 setup.add_command(setup_backups)
@@ -576,7 +583,8 @@ def patch():
 def _fix_prod_perms():
 	"Fix permissions if supervisor processes were run as root"
 	if os.path.exists("config/supervisor.conf"):
-		exec_cmd("supervisorctl stop frappe:")
+		bench_name = get_bench_name(bench_path=".")
+		exec_cmd("supervisorctl stop {bench_name}-processes:".format(bench_name=bench_name))
 
 	fix_prod_setup_perms()
 

bench/config.py

@@ -3,9 +3,10 @@
 import json
 import subprocess
 import shutil
+import socket
 from distutils.spawn import find_executable
 from jinja2 import Environment, PackageLoader
-from .utils import get_sites, get_config, update_config, get_redis_version
+from .utils import get_sites, get_config, update_config, get_redis_version, update_common_site_config, get_bench_name
 
 env = Environment(loader=PackageLoader('bench', 'templates'), trim_blocks=True)
 
@@ -29,7 +30,8 @@ def generate_supervisor_config(bench='.', user=None):
 	sites = get_sites(bench=bench)
 	if not user:
 		user = getpass.getuser()
-	config = get_config()
+
+	config = get_config(bench=bench)
 
 	config = template.render(**{
 		"bench_dir": bench_dir,
@@ -40,7 +42,11 @@ def generate_supervisor_config(bench='.', user=None):
 		"node": find_executable('node') or find_executable('nodejs'),
 		"redis_cache_config": os.path.join(bench_dir, 'config', 'redis_cache.conf'),
 		"redis_async_broker_config": os.path.join(bench_dir, 'config', 'redis_async_broker.conf'),
-		"frappe_version": get_current_frappe_version()
+		"redis_celery_broker_config": os.path.join(bench_dir, 'config', 'redis_celery_broker.conf'),
+		"frappe_version": get_current_frappe_version(),
+		"webserver_port": config.get('webserver_port', 8000),
+		"gunicorn_workers": config.get('gunicorn_workers', 2),
+		"bench_name": get_bench_name(bench)
 	})
 	write_config_file(bench, 'supervisor.conf', config)
 	update_config({'restart_supervisor_on_update': True})
@@ -49,6 +55,27 @@ def get_site_config(site, bench='.'):
 	with open(os.path.join(bench, 'sites', site, 'site_config.json')) as f:
 		return json.load(f)
 
+def generate_common_site_config(bench='.'):
+	'''Generates the default common_site_config.json while a new bench is created'''
+	config = get_config(bench=bench)
+	common_site_config = {}
+
+	for bench_config_field, site_config_field in (
+			("redis_celery_broker_port", "celery_broker"),
+			("redis_async_broker_port", "async_redis_server"),
+			("redis_cache_port", "cache_redis_server")
+		):
+
+		port = config.get(bench_config_field)
+		if config.get(bench_config_field):
+			redis_url = "redis://localhost:{0}".format(port)
+			common_site_config[site_config_field] = redis_url
+
+	# TODO Optionally we need to add the host or domain name in case dns_multitenant is false
+
+	if common_site_config:
+		update_common_site_config(common_site_config, bench=bench)
+
 def get_sites_with_config(bench='.'):
 	sites = get_sites(bench=bench)
 	ret = []
@@ -68,8 +95,9 @@ def generate_nginx_config(bench='.'):
 	sites_dir = os.path.join(bench_dir, "sites")
 	sites = get_sites_with_config(bench=bench)
 	user = getpass.getuser()
+	config = get_config(bench)
 
-	if get_config().get('serve_default_site'):
+	if config.get('serve_default_site'):
 		try:
 			with open("sites/currentsite.txt") as f:
 				default_site = {'name': f.read().strip()}
@@ -80,29 +108,56 @@ def generate_nginx_config(bench='.'):
 
 	config = template.render(**{
 		"sites_dir": sites_dir,
-		"http_timeout": get_config().get("http_timeout", 120),
+		"http_timeout": config.get("http_timeout", 120),
 		"default_site": default_site,
-		"dns_multitenant": get_config().get('dns_multitenant'),
-		"sites": sites
+		"dns_multitenant": config.get('dns_multitenant'),
+		"sites": sites,
+		"webserver_port": config.get('webserver_port', 8000),
+		"socketio_port": config.get('socketio_port', 3000),
+		"bench_name": get_bench_name(bench)
 	})
 	write_config_file(bench, 'nginx.conf', config)
 
+def generate_redis_celery_broker_config(bench='.'):
+	"""Redis that is used for queueing celery tasks"""
+	_generate_redis_config(
+		template_name='redis_celery_broker.conf',
+		context={
+			"port": get_config(bench).get('redis_celery_broker_port', '11311'),
+			"bench_path": os.path.abspath(bench),
+		},
+		bench=bench
+	)
+
+def generate_redis_async_broker_config(bench='.'):
+	"""Redis that is used to do pub/sub"""
+	_generate_redis_config(
+		template_name='redis_async_broker.conf',
+		context={
+			"port": get_config(bench).get('redis_async_broker_port', '12311'),
+		},
+		bench=bench
+	)
+
 def generate_redis_cache_config(bench='.'):
-	template = env.get_template('redis_cache.conf')
-	conf = {
-		"maxmemory": get_config().get('cache_maxmemory', '50'),
-		"port": get_config().get('redis_cache_port', '11311'),
-		"redis_version": get_redis_version()
-	}
-	config = template.render(**conf)
-	write_config_file(bench, 'redis_cache.conf', config)
+	"""Redis that is used and optimized for caching"""
+	config = get_config(bench=bench)
 
+	_generate_redis_config(
+		template_name='redis_cache.conf',
+		context={
+			"maxmemory": config.get('cache_maxmemory', '50'),
+			"port": config.get('redis_cache_port', '13311'),
+			"redis_version": get_redis_version(),
+		},
+		bench=bench
+	)
 
-def generate_redis_async_broker_config(bench='.'):
-	template = env.get_template('redis_async_broker.conf')
-	conf = {
-		"port": get_config().get('redis_async_broker_port', '12311'),
-		"redis_version": get_redis_version()
-	}
-	config = template.render(**conf)
-	write_config_file(bench, 'redis_async_broker.conf', config)
+def _generate_redis_config(template_name, context, bench):
+	template = env.get_template(template_name)
+
+	if "pid_path" not in context:
+		context["pid_path"] = os.path.abspath(os.path.join(bench, "config", "pids"))
+
+	redis_config = template.render(**context)
+	write_config_file(bench, template_name, redis_config)

bench/production_setup.py

@@ -61,6 +61,9 @@ def setup_production(user, bench='.'):
 	else:
 		supervisor_conf_filename = 'frappe.conf'
 
+
+
+
 	os.symlink(os.path.abspath(os.path.join(bench, 'config', 'supervisor.conf')), os.path.join(get_supervisor_confdir(), supervisor_conf_filename))
 	os.symlink(os.path.abspath(os.path.join(bench, 'config', 'nginx.conf')), '/etc/nginx/conf.d/frappe.conf')
 	exec_cmd('supervisorctl reload')

bench/release.py

@@ -20,7 +20,8 @@
 repo_map = {
 	'frappe': 'frappe',
 	'erpnext': 'erpnext',
-	'shopping_cart': 'shopping-cart'
+	'erpnext_shopify': 'erpnext_shopify',
+	'paypal_integration': 'paypal_integration'
 }
 
 def create_release(repo_path, version, remote='origin', develop_branch='develop', master_branch='master'):
@@ -172,7 +173,7 @@ def bump(repo, bump_type, develop='develop', master='master', remote='upstream')
 	print 'Released {tag} for {repo}'.format(tag=tag_name, repo=repo)
 
 def release(repo, bump_type, develop, master):
-	if not get_config().get('release_bench'):
+	if not get_config(".").get('release_bench'):
 		print 'bench not configured to release'
 		sys.exit(1)
 	global github_username, github_password

bench/templates/nginx.conf

@@ -1,12 +1,9 @@
-
-server_names_hash_bucket_size 64;
-
-upstream frappe {
-    server 127.0.0.1:8000 fail_timeout=0;
+upstream {{ bench_name }}-frappe {
+    server 127.0.0.1:{{ webserver_port }} fail_timeout=0;
 }
 
-upstream socketio-server {
-    server 127.0.0.1:3000 fail_timeout=0;
+upstream {{ bench_name}}-socketio-server {
+    server 127.0.0.1:{{ socketio_port }} fail_timeout=0;
 }
 
 {% macro location_block(site, port=80, default=False, server_name=None, sites=None, dns_multitenant=False) -%}
@@ -24,7 +21,7 @@ upstream socketio-server {
 		}
 
 		location /socket.io {
-			proxy_pass http://socketio-server;
+			proxy_pass http://{{ bench_name }}-socketio-server;
 			proxy_http_version 1.1;
 			proxy_set_header Upgrade $http_upgrade;
 			proxy_set_header Connection "upgrade";
@@ -36,20 +33,20 @@ upstream socketio-server {
 		}
 
 		location / {
-			try_files /{{ "$host" if dns_multitenant else site.name }}/public/$uri @magic;
+			try_files /{{ "$host" if dns_multitenant else site.name }}/public/$uri @webserver;
 		}
 
-		location @magic {
+		location @webserver {
 			proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 			proxy_set_header X-Forwarded-Proto $scheme;
 			{% if not dns_multitenant %}
 			proxy_set_header X-Frappe-Site-Name {{ site.name }};
 			{% endif %}
 			proxy_set_header Host $host;
 			proxy_set_header X-Use-X-Accel-Redirect True;
-			proxy_read_timeout {{http_timeout}};
+			proxy_read_timeout {{ http_timeout }};
 			proxy_redirect off;
-			proxy_pass  http://frappe;
+			proxy_pass  http://{{ bench_name }}-frappe;
 		}
 {%- endmacro %}
 

bench/templates/nginx_default.conf

@@ -33,6 +33,8 @@ http {
     #keepalive_timeout  0;
     keepalive_timeout  65;
 
+    server_names_hash_bucket_size 64;
+
     #gzip  on;
 
     index   index.html index.htm;
@@ -41,4 +43,4 @@ http {
     # See http://nginx.org/en/docs/ngx_core_module.html#include
     # for more information.
     include /etc/nginx/conf.d/*.conf;
-}
+}