From c4b3dde7b9aac778e2a9e372a84205c4645aeefa Mon Sep 17 00:00:00 2001 From: Darrin Hodges Date: Wed, 2 May 2018 21:17:36 +1000 Subject: [PATCH 1/9] update Microsoft.Sql/servers api version --- nested/mssql.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nested/mssql.json b/nested/mssql.json index db94aadf..ca7f273c 100644 --- a/nested/mssql.json +++ b/nested/mssql.json @@ -12,7 +12,7 @@ "resources": [ { "type": "Microsoft.Sql/servers", - "apiVersion": "2015-05-01-preview", + "apiVersion": "2014-04-01", "kind": "", "location": "[resourceGroup().location]", "name": "[parameters('moodleCommon').serverName]", From 673593a00d2d94e538cf62b54aebf18a23fa3f88 Mon Sep 17 00:00:00 2001 From: Darrin Hodges Date: Wed, 2 May 2018 21:23:43 +1000 Subject: [PATCH 2/9] replace with simplified code segment --- nested/recoveryservicesEnlist.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nested/recoveryservicesEnlist.json b/nested/recoveryservicesEnlist.json index 4b9cccce..e4d0cc76 100644 --- a/nested/recoveryservicesEnlist.json +++ b/nested/recoveryservicesEnlist.json @@ -24,7 +24,7 @@ "properties": { "policyId": "[resourceId('Microsoft.RecoveryServices/vaults/backupPolicies',parameters('moodleCommon').vaultName,parameters('moodleCommon').policyName )]", "protectedItemType": "[variables('v2VmType')]", - "sourceResourceId": "[resourceId(subscription().subscriptionId,resourceGroup().name,'Microsoft.Compute/virtualMachines',parameters('vmName'))]" + "sourceResourceId": "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]" } } ], From 4a2d2228fab0c01df7eb0d99d39af1786b005ff1 Mon Sep 17 00:00:00 2001 From: Darrin Hodges Date: Wed, 2 May 2018 21:27:47 +1000 Subject: [PATCH 3/9] remove unused vars and resources --- nested/webvmss.json | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/nested/webvmss.json b/nested/webvmss.json index 80ee6062..488e09d6 100644 --- a/nested/webvmss.json +++ b/nested/webvmss.json @@ -10,16 +10,6 @@ } }, "resources": [ - { - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2017-06-01", - "location": "[resourceGroup().location]", - "name": "[parameters('moodleCommon').vmssdStorageAccounttName]", - "kind": "Storage", - "sku": { - "name": "Standard_LRS" - } - }, { "type": "Microsoft.Compute/virtualMachineScaleSets", "apiVersion": "2017-03-30", @@ -183,7 +173,6 @@ } ], "variables": { - "dstorID": "[resourceId('Microsoft.Storage/storageAccounts',parameters('moodleCommon').vmssdStorageAccounttName)]", "extBeID": "[concat(variables('extLbID'),'/backendAddressPools/',parameters('moodleCommon').extBeName)]", "extFeID": "[concat(variables('extLbID'),'/frontendIPConfigurations/',parameters('moodleCommon').extFeName)]", "extLbID": "[resourceId('Microsoft.Network/loadBalancers',parameters('moodleCommon').lbName)]", From e65f4e3a0fa10476cac6f3ea09d827e6cd9b238c Mon Sep 17 00:00:00 2001 From: Darrin Hodges Date: Wed, 2 May 2018 21:38:32 +1000 Subject: [PATCH 4/9] removed duplicate var 'webServerType' --- azuredeploy.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index 6e1d0264..cafe751f 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -753,8 +753,7 @@ "webServerSetupScriptFilename": "setup_webserver.sh", "webServerType": "[parameters('webServerType')]", "serviceObjective": "[parameters('serviceObjective')]", - "msDbSize": "[parameters('msDbSize')]", - "webServerType": "[parameters('webServerType')]" + "msDbSize": "[parameters('msDbSize')]" }, "octets": "[split(parameters('vNetAddressSpace'), '.')]", "resourceprefix": "[substring(uniqueString(resourceGroup().id, deployment().name), 3, 6)]" From 23f10b13a57571995d5ff0dd88322225b5ae8baa Mon Sep 17 00:00:00 2001 From: Darrin Hodges Date: Wed, 2 May 2018 22:02:59 +1000 Subject: [PATCH 5/9] Revert "removed duplicate var 'webServerType'" Altered wrong template. This reverts commit e65f4e3a0fa10476cac6f3ea09d827e6cd9b238c. --- azuredeploy.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azuredeploy.json b/azuredeploy.json index cafe751f..6e1d0264 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -753,7 +753,8 @@ "webServerSetupScriptFilename": "setup_webserver.sh", "webServerType": "[parameters('webServerType')]", "serviceObjective": "[parameters('serviceObjective')]", - "msDbSize": "[parameters('msDbSize')]" + "msDbSize": "[parameters('msDbSize')]", + "webServerType": "[parameters('webServerType')]" }, "octets": "[split(parameters('vNetAddressSpace'), '.')]", "resourceprefix": "[substring(uniqueString(resourceGroup().id, deployment().name), 3, 6)]" From fd8d7bd5cbfc4ff82f246a871e043d1ad48b2589 Mon Sep 17 00:00:00 2001 From: Darrin Hodges Date: Wed, 2 May 2018 22:03:36 +1000 Subject: [PATCH 6/9] Revert "remove unused vars and resources" Altered wrong template. This reverts commit 4a2d2228fab0c01df7eb0d99d39af1786b005ff1. --- nested/webvmss.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nested/webvmss.json b/nested/webvmss.json index 488e09d6..80ee6062 100644 --- a/nested/webvmss.json +++ b/nested/webvmss.json @@ -10,6 +10,16 @@ } }, "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2017-06-01", + "location": "[resourceGroup().location]", + "name": "[parameters('moodleCommon').vmssdStorageAccounttName]", + "kind": "Storage", + "sku": { + "name": "Standard_LRS" + } + }, { "type": "Microsoft.Compute/virtualMachineScaleSets", "apiVersion": "2017-03-30", @@ -173,6 +183,7 @@ } ], "variables": { + "dstorID": "[resourceId('Microsoft.Storage/storageAccounts',parameters('moodleCommon').vmssdStorageAccounttName)]", "extBeID": "[concat(variables('extLbID'),'/backendAddressPools/',parameters('moodleCommon').extBeName)]", "extFeID": "[concat(variables('extLbID'),'/frontendIPConfigurations/',parameters('moodleCommon').extFeName)]", "extLbID": "[resourceId('Microsoft.Network/loadBalancers',parameters('moodleCommon').lbName)]", From 73ee225e73086396992c8742ac0529f5ba86ce8b Mon Sep 17 00:00:00 2001 From: Darrin Hodges Date: Wed, 2 May 2018 22:03:58 +1000 Subject: [PATCH 7/9] Revert "replace with simplified code segment" Altered wrong template. This reverts commit 673593a00d2d94e538cf62b54aebf18a23fa3f88. --- nested/recoveryservicesEnlist.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nested/recoveryservicesEnlist.json b/nested/recoveryservicesEnlist.json index e4d0cc76..4b9cccce 100644 --- a/nested/recoveryservicesEnlist.json +++ b/nested/recoveryservicesEnlist.json @@ -24,7 +24,7 @@ "properties": { "policyId": "[resourceId('Microsoft.RecoveryServices/vaults/backupPolicies',parameters('moodleCommon').vaultName,parameters('moodleCommon').policyName )]", "protectedItemType": "[variables('v2VmType')]", - "sourceResourceId": "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]" + "sourceResourceId": "[resourceId(subscription().subscriptionId,resourceGroup().name,'Microsoft.Compute/virtualMachines',parameters('vmName'))]" } } ], From 5d724c5ba32a4655dcb5a5e89e549ba2d5ed9e94 Mon Sep 17 00:00:00 2001 From: Darrin Hodges Date: Wed, 2 May 2018 22:04:23 +1000 Subject: [PATCH 8/9] Revert "update Microsoft.Sql/servers api version" Altered wrong template. This reverts commit c4b3dde7b9aac778e2a9e372a84205c4645aeefa. --- nested/mssql.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nested/mssql.json b/nested/mssql.json index ca7f273c..db94aadf 100644 --- a/nested/mssql.json +++ b/nested/mssql.json @@ -12,7 +12,7 @@ "resources": [ { "type": "Microsoft.Sql/servers", - "apiVersion": "2014-04-01", + "apiVersion": "2015-05-01-preview", "kind": "", "location": "[resourceGroup().location]", "name": "[parameters('moodleCommon').serverName]", From 68baf655344bc9f434bfd1c55daad878b0fdf0b6 Mon Sep 17 00:00:00 2001 From: Darrin Hodges Date: Fri, 11 May 2018 07:29:41 +1000 Subject: [PATCH 9/9] add altered scripts - created more common functions between them --- scripts/helper_functions.sh | 937 ++++++++++++++++++++++++++++++++++++ scripts/install_moodle.sh | 924 +---------------------------------- scripts/setup_mahara.sh | 510 ++++++++++++++++++++ scripts/setup_webserver.sh | 495 +------------------ 4 files changed, 1456 insertions(+), 1410 deletions(-) create mode 100644 scripts/setup_mahara.sh diff --git a/scripts/helper_functions.sh b/scripts/helper_functions.sh index 746ba003..a880a71d 100644 --- a/scripts/helper_functions.sh +++ b/scripts/helper_functions.sh @@ -27,6 +27,7 @@ function install_php_sql_driver } + function check_fileServerType_param { local fileServerType=$1 @@ -211,6 +212,942 @@ function configure_nfs_client_and_mount { mount ${MOUNTPOINT} } +# Long fail2ban config file generation code moved here. +function create_fail2ban_configuration +{ + # make sure system does automatic updates and fail2ban + sudo apt-get -y update + sudo apt-get -y install unattended-upgrades fail2ban + + # configure fail2ban + cat < /etc/fail2ban/jail.conf +# Fail2Ban configuration file. +# +# This file was composed for Debian systems from the original one +# provided now under /usr/share/doc/fail2ban/examples/jail.conf +# for additional examples. +# +# Comments: use '#' for comment lines and ';' for inline comments +# +# To avoid merges during upgrades DO NOT MODIFY THIS FILE +# and rather provide your changes in /etc/fail2ban/jail.local +# + +# The DEFAULT allows a global definition of the options. They can be overridden +# in each jail afterwards. + +[DEFAULT] + +# "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not +# ban a host which matches an address in this list. Several addresses can be +# defined using space separator. +ignoreip = 127.0.0.1/8 + +# "bantime" is the number of seconds that a host is banned. +bantime = 600 + +# A host is banned if it has generated "maxretry" during the last "findtime" +# seconds. +findtime = 600 +maxretry = 3 + +# "backend" specifies the backend used to get files modification. +# Available options are "pyinotify", "gamin", "polling" and "auto". +# This option can be overridden in each jail as well. +# +# pyinotify: requires pyinotify (a file alteration monitor) to be installed. +# If pyinotify is not installed, Fail2ban will use auto. +# gamin: requires Gamin (a file alteration monitor) to be installed. +# If Gamin is not installed, Fail2ban will use auto. +# polling: uses a polling algorithm which does not require external libraries. +# auto: will try to use the following backends, in order: +# pyinotify, gamin, polling. +backend = auto + +# "usedns" specifies if jails should trust hostnames in logs, +# warn when reverse DNS lookups are performed, or ignore all hostnames in logs +# +# yes: if a hostname is encountered, a reverse DNS lookup will be performed. +# warn: if a hostname is encountered, a reverse DNS lookup will be performed, +# but it will be logged as a warning. +# no: if a hostname is encountered, will not be used for banning, +# but it will be logged as info. +usedns = warn + +# +# Destination email address used solely for the interpolations in +# jail.{conf,local} configuration files. +destemail = root@localhost + +# +# Name of the sender for mta actions +sendername = Fail2Ban + +# +# ACTIONS +# + +# Default banning action (e.g. iptables, iptables-new, +# iptables-multiport, shorewall, etc) It is used to define +# action_* variables. Can be overridden globally or per +# section within jail.local file +banaction = iptables-multiport + +# email action. Since 0.8.1 upstream fail2ban uses sendmail +# MTA for the mailing. Change mta configuration parameter to mail +# if you want to revert to conventional 'mail'. +mta = sendmail + +# Default protocol +protocol = tcp + +# Specify chain where jumps would need to be added in iptables-* actions +chain = INPUT + +# +# Action shortcuts. To be used to define action parameter + +# The simplest action to take: ban only +action_ = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] + +# ban & send an e-mail with whois report to the destemail. +action_mw = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] + %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s", sendername="%(sendername)s"] + +# ban & send an e-mail with whois report and relevant log lines +# to the destemail. +action_mwl = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] + %(mta)s-whois-lines[name=%(__name__)s, dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s", sendername="%(sendername)s"] + +# Choose default action. To change, just override value of 'action' with the +# interpolation to the chosen action shortcut (e.g. action_mw, action_mwl, etc) in jail.local +# globally (section [DEFAULT]) or per specific section +action = %(action_)s + +# +# JAILS +# + +# Next jails corresponds to the standard configuration in Fail2ban 0.6 which +# was shipped in Debian. Enable any defined here jail by including +# +# [SECTION_NAME] +# enabled = true + +# +# in /etc/fail2ban/jail.local. +# +# Optionally you may override any other parameter (e.g. banaction, +# action, port, logpath, etc) in that section within jail.local + +[ssh] + +enabled = true +port = ssh +filter = sshd +logpath = /var/log/auth.log +maxretry = 6 + +[dropbear] + +enabled = false +port = ssh +filter = dropbear +logpath = /var/log/auth.log +maxretry = 6 + +# Generic filter for pam. Has to be used with action which bans all ports +# such as iptables-allports, shorewall +[pam-generic] + +enabled = false +# pam-generic filter can be customized to monitor specific subset of 'tty's +filter = pam-generic +# port actually must be irrelevant but lets leave it all for some possible uses +port = all +banaction = iptables-allports +port = anyport +logpath = /var/log/auth.log +maxretry = 6 + +[xinetd-fail] + +enabled = false +filter = xinetd-fail +port = all +banaction = iptables-multiport-log +logpath = /var/log/daemon.log +maxretry = 2 + + +[ssh-ddos] + +enabled = false +port = ssh +filter = sshd-ddos +logpath = /var/log/auth.log +maxretry = 6 + + +# Here we use blackhole routes for not requiring any additional kernel support +# to store large volumes of banned IPs + +[ssh-route] + +enabled = false +filter = sshd +action = route +logpath = /var/log/sshd.log +maxretry = 6 + +# Here we use a combination of Netfilter/Iptables and IPsets +# for storing large volumes of banned IPs +# +# IPset comes in two versions. See ipset -V for which one to use +# requires the ipset package and kernel support. +[ssh-iptables-ipset4] + +enabled = false +port = ssh +filter = sshd +banaction = iptables-ipset-proto4 +logpath = /var/log/sshd.log +maxretry = 6 + +[ssh-iptables-ipset6] + +enabled = false +port = ssh +filter = sshd +banaction = iptables-ipset-proto6 +logpath = /var/log/sshd.log +maxretry = 6 + + +# +# HTTP servers +# + +[apache] + +enabled = false +port = http,https +filter = apache-auth +logpath = /var/log/apache*/*error.log +maxretry = 6 + +# default action is now multiport, so apache-multiport jail was left +# for compatibility with previous (<0.7.6-2) releases +[apache-multiport] + +enabled = false +port = http,https +filter = apache-auth +logpath = /var/log/apache*/*error.log +maxretry = 6 + +[apache-noscript] + +enabled = false +port = http,https +filter = apache-noscript +logpath = /var/log/apache*/*error.log +maxretry = 6 + +[apache-overflows] + +enabled = false +port = http,https +filter = apache-overflows +logpath = /var/log/apache*/*error.log +maxretry = 2 + +# Ban attackers that try to use PHP's URL-fopen() functionality +# through GET/POST variables. - Experimental, with more than a year +# of usage in production environments. + +[php-url-fopen] + +enabled = false +port = http,https +filter = php-url-fopen +logpath = /var/www/*/logs/access_log + +# A simple PHP-fastcgi jail which works with lighttpd. +# If you run a lighttpd server, then you probably will +# find these kinds of messages in your error_log: +# ALERT – tried to register forbidden variable ‘GLOBALS’ +# through GET variables (attacker '1.2.3.4', file '/var/www/default/htdocs/index.php') + +[lighttpd-fastcgi] + +enabled = false +port = http,https +filter = lighttpd-fastcgi +logpath = /var/log/lighttpd/error.log + +# Same as above for mod_auth +# It catches wrong authentifications + +[lighttpd-auth] + +enabled = false +port = http,https +filter = suhosin +logpath = /var/log/lighttpd/error.log + +[nginx-http-auth] + +enabled = false +filter = nginx-http-auth +port = http,https +logpath = /var/log/nginx/error.log + +# Monitor roundcube server + +[roundcube-auth] + +enabled = false +filter = roundcube-auth +port = http,https +logpath = /var/log/roundcube/userlogins + + +[sogo-auth] + +enabled = false +filter = sogo-auth +port = http, https +# without proxy this would be: +# port = 20000 +logpath = /var/log/sogo/sogo.log + + +# +# FTP servers +# + +[vsftpd] + +enabled = false +port = ftp,ftp-data,ftps,ftps-data +filter = vsftpd +logpath = /var/log/vsftpd.log +# or overwrite it in jails.local to be +# logpath = /var/log/auth.log +# if you want to rely on PAM failed login attempts +# vsftpd's failregex should match both of those formats +maxretry = 6 + + +[proftpd] + +enabled = false +port = ftp,ftp-data,ftps,ftps-data +filter = proftpd +logpath = /var/log/proftpd/proftpd.log +maxretry = 6 + + +[pure-ftpd] + +enabled = false +port = ftp,ftp-data,ftps,ftps-data +filter = pure-ftpd +logpath = /var/log/syslog +maxretry = 6 + + +[wuftpd] + +enabled = false +port = ftp,ftp-data,ftps,ftps-data +filter = wuftpd +logpath = /var/log/syslog +maxretry = 6 + + +# +# Mail servers +# + +[postfix] + +enabled = false +port = smtp,ssmtp,submission +filter = postfix +logpath = /var/log/mail.log + + +[couriersmtp] + +enabled = false +port = smtp,ssmtp,submission +filter = couriersmtp +logpath = /var/log/mail.log + + +# +# Mail servers authenticators: might be used for smtp,ftp,imap servers, so +# all relevant ports get banned +# + +[courierauth] + +enabled = false +port = smtp,ssmtp,submission,imap2,imap3,imaps,pop3,pop3s +filter = courierlogin +logpath = /var/log/mail.log + + +[sasl] + +enabled = false +port = smtp,ssmtp,submission,imap2,imap3,imaps,pop3,pop3s +filter = postfix-sasl +# You might consider monitoring /var/log/mail.warn instead if you are +# running postfix since it would provide the same log lines at the +# "warn" level but overall at the smaller filesize. +logpath = /var/log/mail.log + +[dovecot] + +enabled = false +port = smtp,ssmtp,submission,imap2,imap3,imaps,pop3,pop3s +filter = dovecot +logpath = /var/log/mail.log + +# To log wrong MySQL access attempts add to /etc/my.cnf: +# log-error=/var/log/mysqld.log +# log-warning = 2 +[mysqld-auth] + +enabled = false +filter = mysqld-auth +port = 3306 +logpath = /var/log/mysqld.log + + +# DNS Servers + + +# These jails block attacks against named (bind9). By default, logging is off +# with bind9 installation. You will need something like this: +# +# logging { +# channel security_file { +# file "/var/log/named/security.log" versions 3 size 30m; +# severity dynamic; +# print-time yes; +# }; +# category security { +# security_file; +# }; +# }; +# +# in your named.conf to provide proper logging + +# !!! WARNING !!! +# Since UDP is connection-less protocol, spoofing of IP and imitation +# of illegal actions is way too simple. Thus enabling of this filter +# might provide an easy way for implementing a DoS against a chosen +# victim. See +# http://nion.modprobe.de/blog/archives/690-fail2ban-+-dns-fail.html +# Please DO NOT USE this jail unless you know what you are doing. +#[named-refused-udp] +# +#enabled = false +#port = domain,953 +#protocol = udp +#filter = named-refused +#logpath = /var/log/named/security.log + +[named-refused-tcp] + +enabled = false +port = domain,953 +protocol = tcp +filter = named-refused +logpath = /var/log/named/security.log + +# Multiple jails, 1 per protocol, are necessary ATM: +# see https://github.com/fail2ban/fail2ban/issues/37 +[asterisk-tcp] + +enabled = false +filter = asterisk +port = 5060,5061 +protocol = tcp +logpath = /var/log/asterisk/messages + +[asterisk-udp] + +enabled = false +filter = asterisk +port = 5060,5061 +protocol = udp +logpath = /var/log/asterisk/messages + + +# Jail for more extended banning of persistent abusers +# !!! WARNING !!! +# Make sure that your loglevel specified in fail2ban.conf/.local +# is not at DEBUG level -- which might then cause fail2ban to fall into +# an infinite loop constantly feeding itself with non-informative lines +[recidive] + +enabled = false +filter = recidive +logpath = /var/log/fail2ban.log +action = iptables-allports[name=recidive] + sendmail-whois-lines[name=recidive, logpath=/var/log/fail2ban.log] +bantime = 604800 ; 1 week +findtime = 86400 ; 1 day +maxretry = 5 +EOF +} + +# Long nginx config file generation code moved here +function create_nginx_configuration +{ + + local siteFQDN=$1 + # Build nginx config + cat < /etc/nginx/nginx.conf +user www-data; +worker_processes 2; +pid /run/nginx.pid; + +events { + worker_connections 768; +} + +http { + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + client_max_body_size 0; + proxy_max_temp_file_size 0; + server_names_hash_bucket_size 128; + fastcgi_buffers 16 16k; + fastcgi_buffer_size 32k; + proxy_buffering off; + include /etc/nginx/mime.types; + default_type application/octet-stream; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + set_real_ip_from 127.0.0.1; + real_ip_header X-Forwarded-For; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE + ssl_prefer_server_ciphers on; + + gzip on; + gzip_disable "msie6"; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + + map \$http_x_forwarded_proto \$fastcgi_https { + default \$https; + http ''; + https on; + } + + log_format moodle_combined '\$remote_addr - \$upstream_http_x_moodleuser [\$time_local] ' + '"\$request" \$status \$body_bytes_sent ' + '"\$http_referer" "\$http_user_agent"'; + + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} +EOF + + cat <> /etc/nginx/sites-enabled/${siteFQDN}.conf +server { + listen 81 default; + server_name ${siteFQDN}; + root /moodle/html/moodle; + index index.php index.html index.htm; + + # Log to syslog + error_log syslog:server=localhost,facility=local1,severity=error,tag=moodle; + access_log syslog:server=localhost,facility=local1,severity=notice,tag=moodle moodle_combined; + + # Log XFF IP instead of varnish + set_real_ip_from 10.0.0.0/8; + set_real_ip_from 127.0.0.1; + set_real_ip_from 172.16.0.0/12; + set_real_ip_from 192.168.0.0/16; + real_ip_header X-Forwarded-For; + real_ip_recursive on; + + + # Redirect to https + if (\$http_x_forwarded_proto != https) { + return 301 https://\$server_name\$request_uri; + } + rewrite ^/(.*\.php)(/)(.*)$ /\$1?file=/\$3 last; + + + # Filter out php-fpm status page + location ~ ^/server-status { + return 404; + } + + location / { + try_files \$uri \$uri/index.php?\$query_string; + } + + location ~ [^/]\.php(/|$) { + fastcgi_split_path_info ^(.+?\.php)(/.*)$; + if (!-f \$document_root\$fastcgi_script_name) { + return 404; + } + + fastcgi_buffers 16 16k; + fastcgi_buffer_size 32k; + fastcgi_param SCRIPT_FILENAME \$document_root\$fastcgi_script_name; + fastcgi_pass unix:/run/php/php7.0-fpm.sock; + fastcgi_read_timeout 3600; + fastcgi_index index.php; + include fastcgi_params; + } +} + +server { + listen 443 ssl; + root /moodle/html/moodle; + index index.php index.html index.htm; + + ssl on; + ssl_certificate /moodle/certs/nginx.crt; + ssl_certificate_key /moodle/certs/nginx.key; + + # Log to syslog + error_log syslog:server=localhost,facility=local1,severity=error,tag=moodle; + access_log syslog:server=localhost,facility=local1,severity=notice,tag=moodle moodle_combined; + + # Log XFF IP instead of varnish + set_real_ip_from 10.0.0.0/8; + set_real_ip_from 127.0.0.1; + set_real_ip_from 172.16.0.0/12; + set_real_ip_from 192.168.0.0/16; + real_ip_header X-Forwarded-For; + real_ip_recursive on; + + location / { + proxy_set_header Host \$host; + proxy_set_header HTTP_REFERER \$http_referer; + proxy_set_header X-Forwarded-Host \$host; + proxy_set_header X-Forwarded-Server \$host; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_pass http://localhost:80; + } +} +EOF + + echo -e "Generating SSL self-signed certificate" + openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /moodle/certs/nginx.key -out /moodle/certs/nginx.crt -subj "/C=BR/ST=SP/L=SaoPaulo/O=IT/CN=$siteFQDN" + + # php config + PhpIni=/etc/php/7.0/fpm/php.ini + sed -i "s/memory_limit.*/memory_limit = 512M/" $PhpIni + sed -i "s/max_execution_time.*/max_execution_time = 18000/" $PhpIni + sed -i "s/max_input_vars.*/max_input_vars = 100000/" $PhpIni + sed -i "s/max_input_time.*/max_input_time = 600/" $PhpIni + sed -i "s/upload_max_filesize.*/upload_max_filesize = 1024M/" $PhpIni + sed -i "s/post_max_size.*/post_max_size = 1056M/" $PhpIni + sed -i "s/;opcache.use_cwd.*/opcache.use_cwd = 1/" $PhpIni + sed -i "s/;opcache.validate_timestamps.*/opcache.validate_timestamps = 1/" $PhpIni + sed -i "s/;opcache.save_comments.*/opcache.save_comments = 1/" $PhpIni + sed -i "s/;opcache.enable_file_override.*/opcache.enable_file_override = 0/" $PhpIni + sed -i "s/;opcache.enable.*/opcache.enable = 1/" $PhpIni + sed -i "s/;opcache.memory_consumption.*/opcache.memory_consumption = 256/" $PhpIni + sed -i "s/;opcache.max_accelerated_files.*/opcache.max_accelerated_files = 8000/" $PhpIni + + # fpm config - overload this + cat < /etc/php/7.0/fpm/pool.d/www.conf +[www] +user = www-data +group = www-data +listen = /run/php/php7.0-fpm.sock +listen.owner = www-data +listen.group = www-data +pm = dynamic +pm.max_children = 3000 +pm.start_servers = 20 +pm.min_spare_servers = 22 +pm.max_spare_servers = 30 +EOF + + # Remove the default site. Moodle is the only site we want + rm -f /etc/nginx/sites-enabled/default + + # restart Nginx + sudo service nginx restart +} + +# Long Varnish configuration file generation code moved here +function create_varnish_configuration +{ + cat <> /etc/varnish/moodle.vcl +vcl 4.0; + +import std; +import directors; +backend default { + .host = "localhost"; + .port = "81"; + .first_byte_timeout = 3600s; + .connect_timeout = 600s; + .between_bytes_timeout = 600s; +} + +sub vcl_recv { + # Varnish does not support SPDY or HTTP/2.0 untill we upgrade to Varnish 5.0 + if (req.method == "PRI") { + return (synth(405)); + } + + if (req.restarts == 0) { + if (req.http.X-Forwarded-For) { + set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip; + } else { + set req.http.X-Forwarded-For = client.ip; + } + } + + # Non-RFC2616 or CONNECT HTTP requests methods filtered. Pipe requests directly to backend + if (req.method != "GET" && + req.method != "HEAD" && + req.method != "PUT" && + req.method != "POST" && + req.method != "TRACE" && + req.method != "OPTIONS" && + req.method != "DELETE") { + return (pipe); + } + + # Varnish don't mess with healthchecks + if (req.url ~ "^/admin/tool/heartbeat" || req.url ~ "^/healthcheck.php") + { + return (pass); + } + + # Pipe requests to backup.php straight to backend - prevents problem with progress bar long polling 503 problem + # This is here because backup.php is POSTing to itself - Filter before !GET&&!HEAD + if (req.url ~ "^/backup/backup.php") + { + return (pipe); + } + + # Varnish only deals with GET and HEAD by default. If request method is not GET or HEAD, pass request to backend + if (req.method != "GET" && req.method != "HEAD") { + return (pass); + } + + ### Rules for Moodle and Totara sites ### + # Moodle doesn't require Cookie to serve following assets. Remove Cookie header from request, so it will be looked up. + if ( req.url ~ "^/altlogin/.+/.+\.(png|jpg|jpeg|gif|css|js|webp)$" || + req.url ~ "^/pix/.+\.(png|jpg|jpeg|gif)$" || + req.url ~ "^/theme/font.php" || + req.url ~ "^/theme/image.php" || + req.url ~ "^/theme/javascript.php" || + req.url ~ "^/theme/jquery.php" || + req.url ~ "^/theme/styles.php" || + req.url ~ "^/theme/yui" || + req.url ~ "^/lib/javascript.php/-1/" || + req.url ~ "^/lib/requirejs.php/-1/" + ) + { + set req.http.X-Long-TTL = "86400"; + unset req.http.Cookie; + return(hash); + } + + # Perform lookup for selected assets that we know are static but Moodle still needs a Cookie + if( req.url ~ "^/theme/.+\.(png|jpg|jpeg|gif|css|js|webp)" || + req.url ~ "^/lib/.+\.(png|jpg|jpeg|gif|css|js|webp)" || + req.url ~ "^/pluginfile.php/[0-9]+/course/overviewfiles/.+\.(?i)(png|jpg)$" + ) + { + # Set internal temporary header, based on which we will do things in vcl_backend_response + set req.http.X-Long-TTL = "86400"; + return (hash); + } + + # Serve requests to SCORM checknet.txt from varnish. Have to remove get parameters. Response body always contains "1" + if ( req.url ~ "^/lib/yui/build/moodle-core-checknet/assets/checknet.txt" ) + { + set req.url = regsub(req.url, "(.*)\?.*", "\1"); + unset req.http.Cookie; # Will go to hash anyway at the end of vcl_recv + set req.http.X-Long-TTL = "86400"; + return(hash); + } + + # Requests containing "Cookie" or "Authorization" headers will not be cached + if (req.http.Authorization || req.http.Cookie) { + return (pass); + } + + # Almost everything in Moodle correctly serves Cache-Control headers, if + # needed, which varnish will honor, but there are some which don't. Rather + # than explicitly finding them all and listing them here we just fail safe + # and don't cache unknown urls that get this far. + return (pass); +} + +sub vcl_backend_response { + # Happens after we have read the response headers from the backend. + # + # Here you clean the response headers, removing silly Set-Cookie headers + # and other mistakes your backend does. + + # We know these assest are static, let's set TTL >0 and allow client caching + if ( beresp.http.Cache-Control && bereq.http.X-Long-TTL && beresp.ttl < std.duration(bereq.http.X-Long-TTL + "s", 1s) && !beresp.http.WWW-Authenticate ) + { # If max-age < defined in X-Long-TTL header + set beresp.http.X-Orig-Pragma = beresp.http.Pragma; unset beresp.http.Pragma; + set beresp.http.X-Orig-Cache-Control = beresp.http.Cache-Control; + set beresp.http.Cache-Control = "public, max-age="+bereq.http.X-Long-TTL+", no-transform"; + set beresp.ttl = std.duration(bereq.http.X-Long-TTL + "s", 1s); + unset bereq.http.X-Long-TTL; + } + else if( !beresp.http.Cache-Control && bereq.http.X-Long-TTL && !beresp.http.WWW-Authenticate ) { + set beresp.http.X-Orig-Pragma = beresp.http.Pragma; unset beresp.http.Pragma; + set beresp.http.Cache-Control = "public, max-age="+bereq.http.X-Long-TTL+", no-transform"; + set beresp.ttl = std.duration(bereq.http.X-Long-TTL + "s", 1s); + unset bereq.http.X-Long-TTL; + } + else { # Don't touch headers if max-age > defined in X-Long-TTL header + unset bereq.http.X-Long-TTL; + } + + # Here we set X-Trace header, prepending it to X-Trace header received from backend. Useful for troubleshooting + if(beresp.http.x-trace && !beresp.was_304) { + set beresp.http.X-Trace = regsub(server.identity, "^([^.]+),?.*$", "\1")+"->"+regsub(beresp.backend.name, "^(.+)\((?:[0-9]{1,3}\.){3}([0-9]{1,3})\)","\1(\2)")+"->"+beresp.http.X-Trace; + } + else { + set beresp.http.X-Trace = regsub(server.identity, "^([^.]+),?.*$", "\1")+"->"+regsub(beresp.backend.name, "^(.+)\((?:[0-9]{1,3}\.){3}([0-9]{1,3})\)","\1(\2)"); + } + + # Gzip JS, CSS is done at the ngnix level doing it here dosen't respect the no buffer requsets + # if (beresp.http.content-type ~ "application/javascript.*" || beresp.http.content-type ~ "text") { + # set beresp.do_gzip = true; + #} +} + +sub vcl_deliver { + + # Revert back to original Cache-Control header before delivery to client + if (resp.http.X-Orig-Cache-Control) + { + set resp.http.Cache-Control = resp.http.X-Orig-Cache-Control; + unset resp.http.X-Orig-Cache-Control; + } + + # Revert back to original Pragma header before delivery to client + if (resp.http.X-Orig-Pragma) + { + set resp.http.Pragma = resp.http.X-Orig-Pragma; + unset resp.http.X-Orig-Pragma; + } + + # (Optional) X-Cache HTTP header will be added to responce, indicating whether object was retrieved from backend, or served from cache + if (obj.hits > 0) { + set resp.http.X-Cache = "HIT"; + } else { + set resp.http.X-Cache = "MISS"; + } + + # Set X-AuthOK header when totara/varnsih authentication succeeded + if (req.http.X-AuthOK) { + set resp.http.X-AuthOK = req.http.X-AuthOK; + } + + # If desired "Via: 1.1 Varnish-v4" response header can be removed from response + unset resp.http.Via; + unset resp.http.Server; + + return(deliver); +} + +sub vcl_backend_error { + # More comprehensive varnish error page. Display time, instance hostname, host header, url for easier troubleshooting. + set beresp.http.Content-Type = "text/html; charset=utf-8"; + set beresp.http.Retry-After = "5"; + synthetic( {" + + + + "} + beresp.status + " " + beresp.reason + {" + + +

Error "} + beresp.status + " " + beresp.reason + {"

+

"} + beresp.reason + {"

+

Guru Meditation:

+

Time: "} + now + {"

+

Node: "} + server.hostname + {"

+

Host: "} + bereq.http.host + {"

+

URL: "} + bereq.url + {"

+

XID: "} + bereq.xid + {"

+
+

Varnish cache server + + + "} ); + return (deliver); +} + +sub vcl_synth { + + #Redirect using '301 - Permanent Redirect', permanent redirect + if (resp.status == 851) { + set resp.http.Location = req.http.x-redir; + set resp.http.X-Varnish-Redirect = true; + set resp.status = 301; + return (deliver); + } + + #Redirect using '302 - Found', temporary redirect + if (resp.status == 852) { + set resp.http.Location = req.http.x-redir; + set resp.http.X-Varnish-Redirect = true; + set resp.status = 302; + return (deliver); + } + + #Redirect using '307 - Temporary Redirect', !GET&&!HEAD requests, dont change method on redirected requests + if (resp.status == 857) { + set resp.http.Location = req.http.x-redir; + set resp.http.X-Varnish-Redirect = true; + set resp.status = 307; + return (deliver); + } + + #Respond with 403 - Forbidden + if (resp.status == 863) { + set resp.http.X-Varnish-Error = true; + set resp.status = 403; + return (deliver); + } +} +EOF + + # Restart Varnish + systemctl daemon-reload + service varnish restart +} + # Long Redis cache Moodle config file generation code moved here function create_redis_configuration_in_moodledata_muc_config_php { diff --git a/scripts/install_moodle.sh b/scripts/install_moodle.sh index ed18131c..45bd71ad 100644 --- a/scripts/install_moodle.sh +++ b/scripts/install_moodle.sh @@ -94,496 +94,8 @@ exit 1 fi - # make sure system does automatic updates and fail2ban - sudo apt-get -y update - sudo apt-get -y install unattended-upgrades fail2ban - - # configure fail2ban - cat < /etc/fail2ban/jail.conf -# Fail2Ban configuration file. -# -# This file was composed for Debian systems from the original one -# provided now under /usr/share/doc/fail2ban/examples/jail.conf -# for additional examples. -# -# Comments: use '#' for comment lines and ';' for inline comments -# -# To avoid merges during upgrades DO NOT MODIFY THIS FILE -# and rather provide your changes in /etc/fail2ban/jail.local -# - -# The DEFAULT allows a global definition of the options. They can be overridden -# in each jail afterwards. - -[DEFAULT] - -# "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not -# ban a host which matches an address in this list. Several addresses can be -# defined using space separator. -ignoreip = 127.0.0.1/8 - -# "bantime" is the number of seconds that a host is banned. -bantime = 600 - -# A host is banned if it has generated "maxretry" during the last "findtime" -# seconds. -findtime = 600 -maxretry = 3 - -# "backend" specifies the backend used to get files modification. -# Available options are "pyinotify", "gamin", "polling" and "auto". -# This option can be overridden in each jail as well. -# -# pyinotify: requires pyinotify (a file alteration monitor) to be installed. -# If pyinotify is not installed, Fail2ban will use auto. -# gamin: requires Gamin (a file alteration monitor) to be installed. -# If Gamin is not installed, Fail2ban will use auto. -# polling: uses a polling algorithm which does not require external libraries. -# auto: will try to use the following backends, in order: -# pyinotify, gamin, polling. -backend = auto - -# "usedns" specifies if jails should trust hostnames in logs, -# warn when reverse DNS lookups are performed, or ignore all hostnames in logs -# -# yes: if a hostname is encountered, a reverse DNS lookup will be performed. -# warn: if a hostname is encountered, a reverse DNS lookup will be performed, -# but it will be logged as a warning. -# no: if a hostname is encountered, will not be used for banning, -# but it will be logged as info. -usedns = warn - -# -# Destination email address used solely for the interpolations in -# jail.{conf,local} configuration files. -destemail = root@localhost - -# -# Name of the sender for mta actions -sendername = Fail2Ban - -# -# ACTIONS -# - -# Default banning action (e.g. iptables, iptables-new, -# iptables-multiport, shorewall, etc) It is used to define -# action_* variables. Can be overridden globally or per -# section within jail.local file -banaction = iptables-multiport - -# email action. Since 0.8.1 upstream fail2ban uses sendmail -# MTA for the mailing. Change mta configuration parameter to mail -# if you want to revert to conventional 'mail'. -mta = sendmail - -# Default protocol -protocol = tcp - -# Specify chain where jumps would need to be added in iptables-* actions -chain = INPUT - -# -# Action shortcuts. To be used to define action parameter - -# The simplest action to take: ban only -action_ = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - -# ban & send an e-mail with whois report to the destemail. -action_mw = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s", sendername="%(sendername)s"] - -# ban & send an e-mail with whois report and relevant log lines -# to the destemail. -action_mwl = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois-lines[name=%(__name__)s, dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s", sendername="%(sendername)s"] - -# Choose default action. To change, just override value of 'action' with the -# interpolation to the chosen action shortcut (e.g. action_mw, action_mwl, etc) in jail.local -# globally (section [DEFAULT]) or per specific section -action = %(action_)s - -# -# JAILS -# - -# Next jails corresponds to the standard configuration in Fail2ban 0.6 which -# was shipped in Debian. Enable any defined here jail by including -# -# [SECTION_NAME] -# enabled = true - -# -# in /etc/fail2ban/jail.local. -# -# Optionally you may override any other parameter (e.g. banaction, -# action, port, logpath, etc) in that section within jail.local - -[ssh] - -enabled = true -port = ssh -filter = sshd -logpath = /var/log/auth.log -maxretry = 6 - -[dropbear] - -enabled = false -port = ssh -filter = dropbear -logpath = /var/log/auth.log -maxretry = 6 - -# Generic filter for pam. Has to be used with action which bans all ports -# such as iptables-allports, shorewall -[pam-generic] - -enabled = false -# pam-generic filter can be customized to monitor specific subset of 'tty's -filter = pam-generic -# port actually must be irrelevant but lets leave it all for some possible uses -port = all -banaction = iptables-allports -port = anyport -logpath = /var/log/auth.log -maxretry = 6 - -[xinetd-fail] - -enabled = false -filter = xinetd-fail -port = all -banaction = iptables-multiport-log -logpath = /var/log/daemon.log -maxretry = 2 - - -[ssh-ddos] - -enabled = false -port = ssh -filter = sshd-ddos -logpath = /var/log/auth.log -maxretry = 6 - - -# Here we use blackhole routes for not requiring any additional kernel support -# to store large volumes of banned IPs - -[ssh-route] - -enabled = false -filter = sshd -action = route -logpath = /var/log/sshd.log -maxretry = 6 - -# Here we use a combination of Netfilter/Iptables and IPsets -# for storing large volumes of banned IPs -# -# IPset comes in two versions. See ipset -V for which one to use -# requires the ipset package and kernel support. -[ssh-iptables-ipset4] - -enabled = false -port = ssh -filter = sshd -banaction = iptables-ipset-proto4 -logpath = /var/log/sshd.log -maxretry = 6 - -[ssh-iptables-ipset6] - -enabled = false -port = ssh -filter = sshd -banaction = iptables-ipset-proto6 -logpath = /var/log/sshd.log -maxretry = 6 - - -# -# HTTP servers -# - -[apache] - -enabled = false -port = http,https -filter = apache-auth -logpath = /var/log/apache*/*error.log -maxretry = 6 - -# default action is now multiport, so apache-multiport jail was left -# for compatibility with previous (<0.7.6-2) releases -[apache-multiport] - -enabled = false -port = http,https -filter = apache-auth -logpath = /var/log/apache*/*error.log -maxretry = 6 - -[apache-noscript] - -enabled = false -port = http,https -filter = apache-noscript -logpath = /var/log/apache*/*error.log -maxretry = 6 - -[apache-overflows] - -enabled = false -port = http,https -filter = apache-overflows -logpath = /var/log/apache*/*error.log -maxretry = 2 - -# Ban attackers that try to use PHP's URL-fopen() functionality -# through GET/POST variables. - Experimental, with more than a year -# of usage in production environments. - -[php-url-fopen] - -enabled = false -port = http,https -filter = php-url-fopen -logpath = /var/www/*/logs/access_log - -# A simple PHP-fastcgi jail which works with lighttpd. -# If you run a lighttpd server, then you probably will -# find these kinds of messages in your error_log: -# ALERT – tried to register forbidden variable ‘GLOBALS’ -# through GET variables (attacker '1.2.3.4', file '/var/www/default/htdocs/index.php') - -[lighttpd-fastcgi] - -enabled = false -port = http,https -filter = lighttpd-fastcgi -logpath = /var/log/lighttpd/error.log - -# Same as above for mod_auth -# It catches wrong authentifications - -[lighttpd-auth] - -enabled = false -port = http,https -filter = suhosin -logpath = /var/log/lighttpd/error.log - -[nginx-http-auth] - -enabled = false -filter = nginx-http-auth -port = http,https -logpath = /var/log/nginx/error.log - -# Monitor roundcube server - -[roundcube-auth] - -enabled = false -filter = roundcube-auth -port = http,https -logpath = /var/log/roundcube/userlogins - - -[sogo-auth] - -enabled = false -filter = sogo-auth -port = http, https -# without proxy this would be: -# port = 20000 -logpath = /var/log/sogo/sogo.log - - -# -# FTP servers -# - -[vsftpd] - -enabled = false -port = ftp,ftp-data,ftps,ftps-data -filter = vsftpd -logpath = /var/log/vsftpd.log -# or overwrite it in jails.local to be -# logpath = /var/log/auth.log -# if you want to rely on PAM failed login attempts -# vsftpd's failregex should match both of those formats -maxretry = 6 - - -[proftpd] - -enabled = false -port = ftp,ftp-data,ftps,ftps-data -filter = proftpd -logpath = /var/log/proftpd/proftpd.log -maxretry = 6 - - -[pure-ftpd] - -enabled = false -port = ftp,ftp-data,ftps,ftps-data -filter = pure-ftpd -logpath = /var/log/syslog -maxretry = 6 - - -[wuftpd] - -enabled = false -port = ftp,ftp-data,ftps,ftps-data -filter = wuftpd -logpath = /var/log/syslog -maxretry = 6 - - -# -# Mail servers -# - -[postfix] - -enabled = false -port = smtp,ssmtp,submission -filter = postfix -logpath = /var/log/mail.log - - -[couriersmtp] - -enabled = false -port = smtp,ssmtp,submission -filter = couriersmtp -logpath = /var/log/mail.log - - -# -# Mail servers authenticators: might be used for smtp,ftp,imap servers, so -# all relevant ports get banned -# - -[courierauth] - -enabled = false -port = smtp,ssmtp,submission,imap2,imap3,imaps,pop3,pop3s -filter = courierlogin -logpath = /var/log/mail.log - - -[sasl] - -enabled = false -port = smtp,ssmtp,submission,imap2,imap3,imaps,pop3,pop3s -filter = postfix-sasl -# You might consider monitoring /var/log/mail.warn instead if you are -# running postfix since it would provide the same log lines at the -# "warn" level but overall at the smaller filesize. -logpath = /var/log/mail.log - -[dovecot] - -enabled = false -port = smtp,ssmtp,submission,imap2,imap3,imaps,pop3,pop3s -filter = dovecot -logpath = /var/log/mail.log - -# To log wrong MySQL access attempts add to /etc/my.cnf: -# log-error=/var/log/mysqld.log -# log-warning = 2 -[mysqld-auth] - -enabled = false -filter = mysqld-auth -port = 3306 -logpath = /var/log/mysqld.log - - -# DNS Servers - - -# These jails block attacks against named (bind9). By default, logging is off -# with bind9 installation. You will need something like this: -# -# logging { -# channel security_file { -# file "/var/log/named/security.log" versions 3 size 30m; -# severity dynamic; -# print-time yes; -# }; -# category security { -# security_file; -# }; -# }; -# -# in your named.conf to provide proper logging - -# !!! WARNING !!! -# Since UDP is connection-less protocol, spoofing of IP and imitation -# of illegal actions is way too simple. Thus enabling of this filter -# might provide an easy way for implementing a DoS against a chosen -# victim. See -# http://nion.modprobe.de/blog/archives/690-fail2ban-+-dns-fail.html -# Please DO NOT USE this jail unless you know what you are doing. -#[named-refused-udp] -# -#enabled = false -#port = domain,953 -#protocol = udp -#filter = named-refused -#logpath = /var/log/named/security.log - -[named-refused-tcp] - -enabled = false -port = domain,953 -protocol = tcp -filter = named-refused -logpath = /var/log/named/security.log - -# Multiple jails, 1 per protocol, are necessary ATM: -# see https://github.com/fail2ban/fail2ban/issues/37 -[asterisk-tcp] - -enabled = false -filter = asterisk -port = 5060,5061 -protocol = tcp -logpath = /var/log/asterisk/messages - -[asterisk-udp] - -enabled = false -filter = asterisk -port = 5060,5061 -protocol = udp -logpath = /var/log/asterisk/messages - - -# Jail for more extended banning of persistent abusers -# !!! WARNING !!! -# Make sure that your loglevel specified in fail2ban.conf/.local -# is not at DEBUG level -- which might then cause fail2ban to fall into -# an infinite loop constantly feeding itself with non-informative lines -[recidive] - -enabled = false -filter = recidive -logpath = /var/log/fail2ban.log -action = iptables-allports[name=recidive] - sendmail-whois-lines[name=recidive, logpath=/var/log/fail2ban.log] -bantime = 604800 ; 1 week -findtime = 86400 ; 1 day -maxretry = 5 -EOF + #Create fail2ban config + create_fail2ban_configuration # create gluster, nfs or Azure Files mount point mkdir -p /moodle @@ -727,441 +239,15 @@ EOF chmod 755 /tmp/setup-moodle.sh sudo -u www-data /tmp/setup-moodle.sh >> /tmp/setupmoodle.log - # Build nginx config - cat < /etc/nginx/nginx.conf -user www-data; -worker_processes 2; -pid /run/nginx.pid; - -events { - worker_connections 768; -} - -http { - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - client_max_body_size 0; - proxy_max_temp_file_size 0; - server_names_hash_bucket_size 128; - fastcgi_buffers 16 16k; - fastcgi_buffer_size 32k; - proxy_buffering off; - include /etc/nginx/mime.types; - default_type application/octet-stream; - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - set_real_ip_from 127.0.0.1; - real_ip_header X-Forwarded-For; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE - ssl_prefer_server_ciphers on; - - gzip on; - gzip_disable "msie6"; - gzip_vary on; - gzip_proxied any; - gzip_comp_level 6; - gzip_buffers 16 8k; - gzip_http_version 1.1; - gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; - - map \$http_x_forwarded_proto \$fastcgi_https { - default \$https; - http ''; - https on; - } - - log_format moodle_combined '\$remote_addr - \$upstream_http_x_moodleuser [\$time_local] ' - '"\$request" \$status \$body_bytes_sent ' - '"\$http_referer" "\$http_user_agent"'; - - - include /etc/nginx/conf.d/*.conf; - include /etc/nginx/sites-enabled/*; -} -EOF - - cat <> /etc/nginx/sites-enabled/${siteFQDN}.conf -server { - listen 81 default; - server_name ${siteFQDN}; - root /moodle/html/moodle; - index index.php index.html index.htm; - - # Log to syslog - error_log syslog:server=localhost,facility=local1,severity=error,tag=moodle; - access_log syslog:server=localhost,facility=local1,severity=notice,tag=moodle moodle_combined; - - # Log XFF IP instead of varnish - set_real_ip_from 10.0.0.0/8; - set_real_ip_from 127.0.0.1; - set_real_ip_from 172.16.0.0/12; - set_real_ip_from 192.168.0.0/16; - real_ip_header X-Forwarded-For; - real_ip_recursive on; - - - # Redirect to https - if (\$http_x_forwarded_proto != https) { - return 301 https://\$server_name\$request_uri; - } - rewrite ^/(.*\.php)(/)(.*)$ /\$1?file=/\$3 last; - - - # Filter out php-fpm status page - location ~ ^/server-status { - return 404; - } - - location / { - try_files \$uri \$uri/index.php?\$query_string; - } - - location ~ [^/]\.php(/|$) { - fastcgi_split_path_info ^(.+?\.php)(/.*)$; - if (!-f \$document_root\$fastcgi_script_name) { - return 404; - } - - fastcgi_buffers 16 16k; - fastcgi_buffer_size 32k; - fastcgi_param SCRIPT_FILENAME \$document_root\$fastcgi_script_name; - fastcgi_pass unix:/run/php/php7.0-fpm.sock; - fastcgi_read_timeout 3600; - fastcgi_index index.php; - include fastcgi_params; - } -} - -server { - listen 443 ssl; - root /moodle/html/moodle; - index index.php index.html index.htm; - - ssl on; - ssl_certificate /moodle/certs/nginx.crt; - ssl_certificate_key /moodle/certs/nginx.key; - - # Log to syslog - error_log syslog:server=localhost,facility=local1,severity=error,tag=moodle; - access_log syslog:server=localhost,facility=local1,severity=notice,tag=moodle moodle_combined; - - # Log XFF IP instead of varnish - set_real_ip_from 10.0.0.0/8; - set_real_ip_from 127.0.0.1; - set_real_ip_from 172.16.0.0/12; - set_real_ip_from 192.168.0.0/16; - real_ip_header X-Forwarded-For; - real_ip_recursive on; - - location / { - proxy_set_header Host \$host; - proxy_set_header HTTP_REFERER \$http_referer; - proxy_set_header X-Forwarded-Host \$host; - proxy_set_header X-Forwarded-Server \$host; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; - proxy_pass http://localhost:80; - } -} -EOF - - echo -e "Generating SSL self-signed certificate" - openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /moodle/certs/nginx.key -out /moodle/certs/nginx.crt -subj "/C=BR/ST=SP/L=SaoPaulo/O=IT/CN=$siteFQDN" - - # php config - PhpIni=/etc/php/7.0/fpm/php.ini - sed -i "s/memory_limit.*/memory_limit = 512M/" $PhpIni - sed -i "s/max_execution_time.*/max_execution_time = 18000/" $PhpIni - sed -i "s/max_input_vars.*/max_input_vars = 100000/" $PhpIni - sed -i "s/max_input_time.*/max_input_time = 600/" $PhpIni - sed -i "s/upload_max_filesize.*/upload_max_filesize = 1024M/" $PhpIni - sed -i "s/post_max_size.*/post_max_size = 1056M/" $PhpIni - sed -i "s/;opcache.use_cwd.*/opcache.use_cwd = 1/" $PhpIni - sed -i "s/;opcache.validate_timestamps.*/opcache.validate_timestamps = 1/" $PhpIni - sed -i "s/;opcache.save_comments.*/opcache.save_comments = 1/" $PhpIni - sed -i "s/;opcache.enable_file_override.*/opcache.enable_file_override = 0/" $PhpIni - sed -i "s/;opcache.enable.*/opcache.enable = 1/" $PhpIni - sed -i "s/;opcache.memory_consumption.*/opcache.memory_consumption = 256/" $PhpIni - sed -i "s/;opcache.max_accelerated_files.*/opcache.max_accelerated_files = 8000/" $PhpIni - - # fpm config - overload this - cat < /etc/php/7.0/fpm/pool.d/www.conf -[www] -user = www-data -group = www-data -listen = /run/php/php7.0-fpm.sock -listen.owner = www-data -listen.group = www-data -pm = dynamic -pm.max_children = 3000 -pm.start_servers = 20 -pm.min_spare_servers = 22 -pm.max_spare_servers = 30 -EOF - - # Remove the default site. Moodle is the only site we want - rm -f /etc/nginx/sites-enabled/default - - # restart Nginx - sudo service nginx restart + #NGINX / PHP config + create_nginx_configuration ${siteFQDN} # Configure varnish startup for 16.04 VARNISHSTART="ExecStart=\/usr\/sbin\/varnishd -j unix,user=vcache -F -a :80 -T localhost:6082 -f \/etc\/varnish\/moodle.vcl -S \/etc\/varnish\/secret -s malloc,1024m -p thread_pool_min=200 -p thread_pool_max=4000 -p thread_pool_add_delay=2 -p timeout_linger=100 -p timeout_idle=30 -p send_timeout=1800 -p thread_pools=4 -p http_max_hdr=512 -p workspace_backend=512k" sed -i "s/^ExecStart.*/${VARNISHSTART}/" /lib/systemd/system/varnish.service # Configure varnish VCL for moodle - cat <> /etc/varnish/moodle.vcl -vcl 4.0; - -import std; -import directors; -backend default { - .host = "localhost"; - .port = "81"; - .first_byte_timeout = 3600s; - .connect_timeout = 600s; - .between_bytes_timeout = 600s; -} - -sub vcl_recv { - # Varnish does not support SPDY or HTTP/2.0 untill we upgrade to Varnish 5.0 - if (req.method == "PRI") { - return (synth(405)); - } - - if (req.restarts == 0) { - if (req.http.X-Forwarded-For) { - set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip; - } else { - set req.http.X-Forwarded-For = client.ip; - } - } - - # Non-RFC2616 or CONNECT HTTP requests methods filtered. Pipe requests directly to backend - if (req.method != "GET" && - req.method != "HEAD" && - req.method != "PUT" && - req.method != "POST" && - req.method != "TRACE" && - req.method != "OPTIONS" && - req.method != "DELETE") { - return (pipe); - } - - # Varnish don't mess with healthchecks - if (req.url ~ "^/admin/tool/heartbeat" || req.url ~ "^/healthcheck.php") - { - return (pass); - } - - # Pipe requests to backup.php straight to backend - prevents problem with progress bar long polling 503 problem - # This is here because backup.php is POSTing to itself - Filter before !GET&&!HEAD - if (req.url ~ "^/backup/backup.php") - { - return (pipe); - } - - # Varnish only deals with GET and HEAD by default. If request method is not GET or HEAD, pass request to backend - if (req.method != "GET" && req.method != "HEAD") { - return (pass); - } - - ### Rules for Moodle and Totara sites ### - # Moodle doesn't require Cookie to serve following assets. Remove Cookie header from request, so it will be looked up. - if ( req.url ~ "^/altlogin/.+/.+\.(png|jpg|jpeg|gif|css|js|webp)$" || - req.url ~ "^/pix/.+\.(png|jpg|jpeg|gif)$" || - req.url ~ "^/theme/font.php" || - req.url ~ "^/theme/image.php" || - req.url ~ "^/theme/javascript.php" || - req.url ~ "^/theme/jquery.php" || - req.url ~ "^/theme/styles.php" || - req.url ~ "^/theme/yui" || - req.url ~ "^/lib/javascript.php/-1/" || - req.url ~ "^/lib/requirejs.php/-1/" - ) - { - set req.http.X-Long-TTL = "86400"; - unset req.http.Cookie; - return(hash); - } - - # Perform lookup for selected assets that we know are static but Moodle still needs a Cookie - if( req.url ~ "^/theme/.+\.(png|jpg|jpeg|gif|css|js|webp)" || - req.url ~ "^/lib/.+\.(png|jpg|jpeg|gif|css|js|webp)" || - req.url ~ "^/pluginfile.php/[0-9]+/course/overviewfiles/.+\.(?i)(png|jpg)$" - ) - { - # Set internal temporary header, based on which we will do things in vcl_backend_response - set req.http.X-Long-TTL = "86400"; - return (hash); - } - - # Serve requests to SCORM checknet.txt from varnish. Have to remove get parameters. Response body always contains "1" - if ( req.url ~ "^/lib/yui/build/moodle-core-checknet/assets/checknet.txt" ) - { - set req.url = regsub(req.url, "(.*)\?.*", "\1"); - unset req.http.Cookie; # Will go to hash anyway at the end of vcl_recv - set req.http.X-Long-TTL = "86400"; - return(hash); - } - - # Requests containing "Cookie" or "Authorization" headers will not be cached - if (req.http.Authorization || req.http.Cookie) { - return (pass); - } - - # Almost everything in Moodle correctly serves Cache-Control headers, if - # needed, which varnish will honor, but there are some which don't. Rather - # than explicitly finding them all and listing them here we just fail safe - # and don't cache unknown urls that get this far. - return (pass); -} - -sub vcl_backend_response { - # Happens after we have read the response headers from the backend. - # - # Here you clean the response headers, removing silly Set-Cookie headers - # and other mistakes your backend does. - - # We know these assest are static, let's set TTL >0 and allow client caching - if ( beresp.http.Cache-Control && bereq.http.X-Long-TTL && beresp.ttl < std.duration(bereq.http.X-Long-TTL + "s", 1s) && !beresp.http.WWW-Authenticate ) - { # If max-age < defined in X-Long-TTL header - set beresp.http.X-Orig-Pragma = beresp.http.Pragma; unset beresp.http.Pragma; - set beresp.http.X-Orig-Cache-Control = beresp.http.Cache-Control; - set beresp.http.Cache-Control = "public, max-age="+bereq.http.X-Long-TTL+", no-transform"; - set beresp.ttl = std.duration(bereq.http.X-Long-TTL + "s", 1s); - unset bereq.http.X-Long-TTL; - } - else if( !beresp.http.Cache-Control && bereq.http.X-Long-TTL && !beresp.http.WWW-Authenticate ) { - set beresp.http.X-Orig-Pragma = beresp.http.Pragma; unset beresp.http.Pragma; - set beresp.http.Cache-Control = "public, max-age="+bereq.http.X-Long-TTL+", no-transform"; - set beresp.ttl = std.duration(bereq.http.X-Long-TTL + "s", 1s); - unset bereq.http.X-Long-TTL; - } - else { # Don't touch headers if max-age > defined in X-Long-TTL header - unset bereq.http.X-Long-TTL; - } - - # Here we set X-Trace header, prepending it to X-Trace header received from backend. Useful for troubleshooting - if(beresp.http.x-trace && !beresp.was_304) { - set beresp.http.X-Trace = regsub(server.identity, "^([^.]+),?.*$", "\1")+"->"+regsub(beresp.backend.name, "^(.+)\((?:[0-9]{1,3}\.){3}([0-9]{1,3})\)","\1(\2)")+"->"+beresp.http.X-Trace; - } - else { - set beresp.http.X-Trace = regsub(server.identity, "^([^.]+),?.*$", "\1")+"->"+regsub(beresp.backend.name, "^(.+)\((?:[0-9]{1,3}\.){3}([0-9]{1,3})\)","\1(\2)"); - } - - # Gzip JS, CSS is done at the ngnix level doing it here dosen't respect the no buffer requsets - # if (beresp.http.content-type ~ "application/javascript.*" || beresp.http.content-type ~ "text") { - # set beresp.do_gzip = true; - #} -} - -sub vcl_deliver { - - # Revert back to original Cache-Control header before delivery to client - if (resp.http.X-Orig-Cache-Control) - { - set resp.http.Cache-Control = resp.http.X-Orig-Cache-Control; - unset resp.http.X-Orig-Cache-Control; - } - - # Revert back to original Pragma header before delivery to client - if (resp.http.X-Orig-Pragma) - { - set resp.http.Pragma = resp.http.X-Orig-Pragma; - unset resp.http.X-Orig-Pragma; - } - - # (Optional) X-Cache HTTP header will be added to responce, indicating whether object was retrieved from backend, or served from cache - if (obj.hits > 0) { - set resp.http.X-Cache = "HIT"; - } else { - set resp.http.X-Cache = "MISS"; - } - - # Set X-AuthOK header when totara/varnsih authentication succeeded - if (req.http.X-AuthOK) { - set resp.http.X-AuthOK = req.http.X-AuthOK; - } - - # If desired "Via: 1.1 Varnish-v4" response header can be removed from response - unset resp.http.Via; - unset resp.http.Server; - - return(deliver); -} - -sub vcl_backend_error { - # More comprehensive varnish error page. Display time, instance hostname, host header, url for easier troubleshooting. - set beresp.http.Content-Type = "text/html; charset=utf-8"; - set beresp.http.Retry-After = "5"; - synthetic( {" - - - - "} + beresp.status + " " + beresp.reason + {" - - -

Error "} + beresp.status + " " + beresp.reason + {"

-

"} + beresp.reason + {"

-

Guru Meditation:

-

Time: "} + now + {"

-

Node: "} + server.hostname + {"

-

Host: "} + bereq.http.host + {"

-

URL: "} + bereq.url + {"

-

XID: "} + bereq.xid + {"

-
-

Varnish cache server - - - "} ); - return (deliver); -} - -sub vcl_synth { - - #Redirect using '301 - Permanent Redirect', permanent redirect - if (resp.status == 851) { - set resp.http.Location = req.http.x-redir; - set resp.http.X-Varnish-Redirect = true; - set resp.status = 301; - return (deliver); - } - - #Redirect using '302 - Found', temporary redirect - if (resp.status == 852) { - set resp.http.Location = req.http.x-redir; - set resp.http.X-Varnish-Redirect = true; - set resp.status = 302; - return (deliver); - } - - #Redirect using '307 - Temporary Redirect', !GET&&!HEAD requests, dont change method on redirected requests - if (resp.status == 857) { - set resp.http.Location = req.http.x-redir; - set resp.http.X-Varnish-Redirect = true; - set resp.status = 307; - return (deliver); - } - - #Respond with 403 - Forbidden - if (resp.status == 863) { - set resp.http.X-Varnish-Error = true; - set resp.status = 403; - return (deliver); - } -} -EOF - - # Restart Varnish - systemctl daemon-reload - service varnish restart + create_varnish_configuration if [ $dbServerType = "mysql" ]; then mysql -h $mysqlIP -u $mysqladminlogin -p${mysqladminpass} -e "CREATE DATABASE ${moodledbname} CHARACTER SET utf8;" diff --git a/scripts/setup_mahara.sh b/scripts/setup_mahara.sh new file mode 100644 index 00000000..9ef136ac --- /dev/null +++ b/scripts/setup_mahara.sh @@ -0,0 +1,510 @@ +# Custom Script for Linux for Mahara/Azure + +#!/bin/bash + +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +glusterNode=$1 +glusterVolume=$2 +siteFQDN=$3 +syslogserver=$4 + +echo $glusterNode >> /tmp/vars.txt +echo $glusterVolume >> /tmp/vars.txt +echo $siteFQDN >> /tmp/vars.txt + +{ + # make sure the system does automatic update + sudo apt-get -y update + sudo apt-get -y install unattended-upgrades + + # install pre-requisites + sudo apt-get -y install python-software-properties unzip rsyslog + + #configure gluster repository & install gluster client + sudo add-apt-repository ppa:gluster/glusterfs-3.8 -y + sudo apt-get -y update + sudo apt-get -y install glusterfs-client postgresql-client mysql-client git + + # install the base stack + sudo apt-get -y install nginx php-fpm varnish php php-cli php-curl php-zip + + # Mahara requirements + sudo apt-get install -y graphviz aspell php-soap php-json php-redis php-bcmath php-gd php-pgsql php-mysql php-xmlrpc php-intl php-xml php-bz2 php-mbstring + + # Mount gluster fs for /mahara + sudo mkdir -p /mahara + sudo chown www-data /mahara + sudo chmod 770 /mahara + sudo echo -e 'mount -t glusterfs '$glusterNode':/'$glusterVolume' /mahara' + sudo mount -t glusterfs $glusterNode:/$glusterVolume /mahara + sudo echo -e $glusterNode':/'$glusterVolume' /mahara glusterfs defaults,_netdev,log-level=WARNING,log-file=/var/log/gluster.log 0 0' >> /etc/fstab + sudo mount -a + + # Configure syslog to forward + cat <> /etc/rsyslog.conf +\$ModLoad imudp +\$UDPServerRun 514 +EOF + cat <> /etc/rsyslog.d/40-remote.conf +local1.* @${syslogserver}:514 +local2.* @${syslogserver}:514 +EOF + service syslog restart + + # Build nginx config + cat < /etc/nginx/nginx.conf +user www-data; +worker_processes 2; +pid /run/nginx.pid; + +events { + worker_connections 2048; +} + +http { + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + client_max_body_size 0; + proxy_max_temp_file_size 0; + server_names_hash_bucket_size 128; + fastcgi_buffers 16 16k; + fastcgi_buffer_size 32k; + proxy_buffering off; + include /etc/nginx/mime.types; + default_type application/octet-stream; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + set_real_ip_from 127.0.0.1; + real_ip_header X-Forwarded-For; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE + ssl_prefer_server_ciphers on; + + gzip on; + gzip_disable "msie6"; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + + map \$http_x_forwarded_proto \$fastcgi_https { + default \$https; + http ''; + https on; + } + + log_format mahara_combined '\$remote_addr - \$upstream_http_x_maharauser [\$time_local] ' + '"\$request" \$status \$body_bytes_sent ' + '"\$http_referer" "\$http_user_agent"'; + + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} +EOF + + cat <> /etc/nginx/sites-enabled/${siteFQDN}.conf +server { + listen 81 default; + server_name ${siteFQDN}; + root /mahara/html/mahara/htdocs; + index index.php index.html index.htm; + + # Log to syslog + error_log syslog:server=localhost,facility=local1,severity=error,tag=mahara; + access_log syslog:server=localhost,facility=local1,severity=notice,tag=mahara mahara_combined; + + # Log XFF IP instead of varnish + set_real_ip_from 10.0.0.0/8; + set_real_ip_from 127.0.0.1; + set_real_ip_from 172.16.0.0/12; + set_real_ip_from 192.168.0.0/16; + real_ip_header X-Forwarded-For; + real_ip_recursive on; + + + # Redirect to https + if (\$http_x_forwarded_proto != https) { + return 301 https://\$server_name\$request_uri; + } + rewrite ^/(.*\.php)(/)(.*)$ /\$1?file=/\$3 last; + + + # Filter out php-fpm status page + location ~ ^/server-status { + return 404; + } + + location / { + try_files \$uri \$uri/index.php?\$query_string; + } + + location ~ [^/]\.php(/|$) { + fastcgi_split_path_info ^(.+?\.php)(/.*)$; + if (!-f \$document_root\$fastcgi_script_name) { + return 404; + } + + fastcgi_buffers 16 16k; + fastcgi_buffer_size 32k; + fastcgi_param SCRIPT_FILENAME \$document_root\$fastcgi_script_name; + fastcgi_pass unix:/run/php/php7.0-fpm.sock; + fastcgi_read_timeout 3600; + fastcgi_index index.php; + include fastcgi_params; + } +} + +server { + listen 443 ssl; + root /mahara/html/mahara/htdocs; + index index.php index.html index.htm; + + ssl on; + ssl_certificate /mahara/certs/nginx.crt; + ssl_certificate_key /mahara/certs/nginx.key; + + # Log to syslog + error_log syslog:server=localhost,facility=local1,severity=error,tag=mahara; + access_log syslog:server=localhost,facility=local1,severity=notice,tag=mahara mahara_combined; + + # Log XFF IP instead of varnish + set_real_ip_from 10.0.0.0/8; + set_real_ip_from 127.0.0.1; + set_real_ip_from 172.16.0.0/12; + set_real_ip_from 192.168.0.0/16; + real_ip_header X-Forwarded-For; + real_ip_recursive on; + + location / { + proxy_set_header Host \$host; + proxy_set_header HTTP_REFERER \$http_referer; + proxy_set_header X-Forwarded-Host \$host; + proxy_set_header X-Forwarded-Server \$host; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_pass http://localhost:80; + } +} +EOF + + # php config + PhpIni=/etc/php/7.0/fpm/php.ini + sed -i "s/memory_limit.*/memory_limit = 512M/" $PhpIni + sed -i "s/max_execution_time.*/max_execution_time = 18000/" $PhpIni + sed -i "s/max_input_vars.*/max_input_vars = 100000/" $PhpIni + sed -i "s/max_input_time.*/max_input_time = 600/" $PhpIni + sed -i "s/upload_max_filesize.*/upload_max_filesize = 1024M/" $PhpIni + sed -i "s/post_max_size.*/post_max_size = 1056M/" $PhpIni + sed -i "s/;opcache.use_cwd.*/opcache.use_cwd = 1/" $PhpIni + sed -i "s/;opcache.validate_timestamps.*/opcache.validate_timestamps = 1/" $PhpIni + sed -i "s/;opcache.save_comments.*/opcache.save_comments = 1/" $PhpIni + sed -i "s/;opcache.enable_file_override.*/opcache.enable_file_override = 0/" $PhpIni + sed -i "s/;opcache.enable.*/opcache.enable = 1/" $PhpIni + sed -i "s/;opcache.memory_consumption.*/opcache.memory_consumption = 256/" $PhpIni + sed -i "s/;opcache.max_accelerated_files.*/opcache.max_accelerated_files = 8000/" $PhpIni + + # Remove the default site. Mahara is the only site we want + rm -f /etc/nginx/sites-enabled/default + + # restart Nginx + sudo service nginx restart + + # fpm config - overload this + cat < /etc/php/7.0/fpm/pool.d/www.conf +[www] +user = www-data +group = www-data +listen = /run/php/php7.0-fpm.sock +listen.owner = www-data +listen.group = www-data +pm = dynamic +pm.max_children = 3000 +pm.start_servers = 20 +pm.min_spare_servers = 20 +pm.max_spare_servers = 30 +EOF + + # Restart fpm + service php7.0-fpm restart + + # Configure varnish startup for 16.04 + VARNISHSTART="ExecStart=\/usr\/sbin\/varnishd -j unix,user=vcache -F -a :80 -T localhost:6082 -f \/etc\/varnish\/mahara.vcl -S \/etc\/varnish\/secret -s malloc,1024m -p thread_pool_min=200 -p thread_pool_max=4000 -p thread_pool_add_delay=2 -p timeout_linger=100 -p timeout_idle=30 -p send_timeout=1800 -p thread_pools=4 -p http_max_hdr=512 -p workspace_backend=512k" + sed -i "s/^ExecStart.*/${VARNISHSTART}/" /lib/systemd/system/varnish.service + + # Configure varnish VCL for mahara + cat <> /etc/varnish/mahara.vcl +vcl 4.0; + +import std; +import directors; +backend default { + .host = "localhost"; + .port = "81"; + .first_byte_timeout = 3600s; + .connect_timeout = 600s; + .between_bytes_timeout = 600s; +} + +sub vcl_recv { + # Varnish does not support SPDY or HTTP/2.0 untill we upgrade to Varnish 5.0 + if (req.method == "PRI") { + return (synth(405)); + } + + if (req.restarts == 0) { + if (req.http.X-Forwarded-For) { + set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip; + } else { + set req.http.X-Forwarded-For = client.ip; + } + } + + # Non-RFC2616 or CONNECT HTTP requests methods filtered. Pipe requests directly to backend + if (req.method != "GET" && + req.method != "HEAD" && + req.method != "PUT" && + req.method != "POST" && + req.method != "TRACE" && + req.method != "OPTIONS" && + req.method != "DELETE") { + return (pipe); + } + + # Varnish don't mess with healthchecks + if (req.url ~ "^/admin/tool/heartbeat" || req.url ~ "^/healthcheck.php") + { + return (pass); + } + + # Pipe requests to backup.php straight to backend - prevents problem with progress bar long polling 503 problem + # This is here because backup.php is POSTing to itself - Filter before !GET&&!HEAD + if (req.url ~ "^/backup/backup.php") + { + return (pipe); + } + + # Varnish only deals with GET and HEAD by default. If request method is not GET or HEAD, pass request to backend + if (req.method != "GET" && req.method != "HEAD") { + return (pass); + } + + ### Rules for Mahara and Totara sites ### + # Mahara doesn't require Cookie to serve following assets. Remove Cookie header from request, so it will be looked up. + if ( req.url ~ "^/altlogin/.+/.+\.(png|jpg|jpeg|gif|css|js|webp)$" || + req.url ~ "^/pix/.+\.(png|jpg|jpeg|gif)$" || + req.url ~ "^/theme/font.php" || + req.url ~ "^/theme/image.php" || + req.url ~ "^/theme/javascript.php" || + req.url ~ "^/theme/jquery.php" || + req.url ~ "^/theme/styles.php" || + req.url ~ "^/theme/yui" || + req.url ~ "^/lib/javascript.php/-1/" || + req.url ~ "^/lib/requirejs.php/-1/" + ) + { + set req.http.X-Long-TTL = "86400"; + unset req.http.Cookie; + return(hash); + } + + # Perform lookup for selected assets that we know are static but Mahara still needs a Cookie + if( req.url ~ "^/theme/.+\.(png|jpg|jpeg|gif|css|js|webp)" || + req.url ~ "^/lib/.+\.(png|jpg|jpeg|gif|css|js|webp)" || + req.url ~ "^/pluginfile.php/[0-9]+/course/overviewfiles/.+\.(?i)(png|jpg)$" + ) + { + # Set internal temporary header, based on which we will do things in vcl_backend_response + set req.http.X-Long-TTL = "86400"; + return (hash); + } + + # Serve requests to SCORM checknet.txt from varnish. Have to remove get parameters. Response body always contains "1" + if ( req.url ~ "^/lib/yui/build/mahara-core-checknet/assets/checknet.txt" ) + { + set req.url = regsub(req.url, "(.*)\?.*", "\1"); + unset req.http.Cookie; # Will go to hash anyway at the end of vcl_recv + set req.http.X-Long-TTL = "86400"; + return(hash); + } + + # Requests containing "Cookie" or "Authorization" headers will not be cached + if (req.http.Authorization || req.http.Cookie) { + return (pass); + } + + # Almost everything in Mahara correctly serves Cache-Control headers, if + # needed, which varnish will honor, but there are some which don't. Rather + # than explicitly finding them all and listing them here we just fail safe + # and don't cache unknown urls that get this far. + return (pass); +} + +sub vcl_backend_response { + # Happens after we have read the response headers from the backend. + # + # Here you clean the response headers, removing silly Set-Cookie headers + # and other mistakes your backend does. + + # We know these assest are static, let's set TTL >0 and allow client caching + if ( beresp.http.Cache-Control && bereq.http.X-Long-TTL && beresp.ttl < std.duration(bereq.http.X-Long-TTL + "s", 1s) && !beresp.http.WWW-Authenticate ) + { # If max-age < defined in X-Long-TTL header + set beresp.http.X-Orig-Pragma = beresp.http.Pragma; unset beresp.http.Pragma; + set beresp.http.X-Orig-Cache-Control = beresp.http.Cache-Control; + set beresp.http.Cache-Control = "public, max-age="+bereq.http.X-Long-TTL+", no-transform"; + set beresp.ttl = std.duration(bereq.http.X-Long-TTL + "s", 1s); + unset bereq.http.X-Long-TTL; + } + else if( !beresp.http.Cache-Control && bereq.http.X-Long-TTL && !beresp.http.WWW-Authenticate ) { + set beresp.http.X-Orig-Pragma = beresp.http.Pragma; unset beresp.http.Pragma; + set beresp.http.Cache-Control = "public, max-age="+bereq.http.X-Long-TTL+", no-transform"; + set beresp.ttl = std.duration(bereq.http.X-Long-TTL + "s", 1s); + unset bereq.http.X-Long-TTL; + } + else { # Don't touch headers if max-age > defined in X-Long-TTL header + unset bereq.http.X-Long-TTL; + } + + # Here we set X-Trace header, prepending it to X-Trace header received from backend. Useful for troubleshooting + if(beresp.http.x-trace && !beresp.was_304) { + set beresp.http.X-Trace = regsub(server.identity, "^([^.]+),?.*$", "\1")+"->"+regsub(beresp.backend.name, "^(.+)\((?:[0-9]{1,3}\.){3}([0-9]{1,3})\)","\1(\2)")+"->"+beresp.http.X-Trace; + } + else { + set beresp.http.X-Trace = regsub(server.identity, "^([^.]+),?.*$", "\1")+"->"+regsub(beresp.backend.name, "^(.+)\((?:[0-9]{1,3}\.){3}([0-9]{1,3})\)","\1(\2)"); + } + + # Gzip JS, CSS is done at the ngnix level doing it here dosen't respect the no buffer requsets + # if (beresp.http.content-type ~ "application/javascript.*" || beresp.http.content-type ~ "text") { + # set beresp.do_gzip = true; + #} +} + +sub vcl_deliver { + + # Revert back to original Cache-Control header before delivery to client + if (resp.http.X-Orig-Cache-Control) + { + set resp.http.Cache-Control = resp.http.X-Orig-Cache-Control; + unset resp.http.X-Orig-Cache-Control; + } + + # Revert back to original Pragma header before delivery to client + if (resp.http.X-Orig-Pragma) + { + set resp.http.Pragma = resp.http.X-Orig-Pragma; + unset resp.http.X-Orig-Pragma; + } + + # (Optional) X-Cache HTTP header will be added to responce, indicating whether object was retrieved from backend, or served from cache + if (obj.hits > 0) { + set resp.http.X-Cache = "HIT"; + } else { + set resp.http.X-Cache = "MISS"; + } + + # Set X-AuthOK header when totara/varnsih authentication succeeded + if (req.http.X-AuthOK) { + set resp.http.X-AuthOK = req.http.X-AuthOK; + } + + # If desired "Via: 1.1 Varnish-v4" response header can be removed from response + unset resp.http.Via; + unset resp.http.Server; + + return(deliver); +} + +sub vcl_backend_error { + # More comprehensive varnish error page. Display time, instance hostname, host header, url for easier troubleshooting. + set beresp.http.Content-Type = "text/html; charset=utf-8"; + set beresp.http.Retry-After = "5"; + synthetic( {" + + + + "} + beresp.status + " " + beresp.reason + {" + + +

Error "} + beresp.status + " " + beresp.reason + {"

+

"} + beresp.reason + {"

+

Guru Meditation:

+

Time: "} + now + {"

+

Node: "} + server.hostname + {"

+

Host: "} + bereq.http.host + {"

+

URL: "} + bereq.url + {"

+

XID: "} + bereq.xid + {"

+
+

Varnish cache server + + + "} ); + return (deliver); +} + +sub vcl_synth { + + #Redirect using '301 - Permanent Redirect', permanent redirect + if (resp.status == 851) { + set resp.http.Location = req.http.x-redir; + set resp.http.X-Varnish-Redirect = true; + set resp.status = 301; + return (deliver); + } + + #Redirect using '302 - Found', temporary redirect + if (resp.status == 852) { + set resp.http.Location = req.http.x-redir; + set resp.http.X-Varnish-Redirect = true; + set resp.status = 302; + return (deliver); + } + + #Redirect using '307 - Temporary Redirect', !GET&&!HEAD requests, dont change method on redirected requests + if (resp.status == 857) { + set resp.http.Location = req.http.x-redir; + set resp.http.X-Varnish-Redirect = true; + set resp.status = 307; + return (deliver); + } + + #Respond with 403 - Forbidden + if (resp.status == 863) { + set resp.http.X-Varnish-Error = true; + set resp.status = 403; + return (deliver); + } +} +#this is a comment +EOF + + # Restart Varnish + systemctl daemon-reload + service varnish restart + +} > /tmp/setup.log diff --git a/scripts/setup_webserver.sh b/scripts/setup_webserver.sh index f44154b8..d821863a 100644 --- a/scripts/setup_webserver.sh +++ b/scripts/setup_webserver.sh @@ -105,501 +105,14 @@ local2.* @${syslogserver}:514 EOF service syslog restart - # Build nginx config - cat < /etc/nginx/nginx.conf -user www-data; -worker_processes 2; -pid /run/nginx.pid; - -events { - worker_connections 2048; -} - -http { - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - client_max_body_size 0; - proxy_max_temp_file_size 0; - server_names_hash_bucket_size 128; - fastcgi_buffers 16 16k; - fastcgi_buffer_size 32k; - proxy_buffering off; - include /etc/nginx/mime.types; - default_type application/octet-stream; - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - set_real_ip_from 127.0.0.1; - real_ip_header X-Forwarded-For; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE - ssl_prefer_server_ciphers on; - - gzip on; - gzip_disable "msie6"; - gzip_vary on; - gzip_proxied any; - gzip_comp_level 6; - gzip_buffers 16 8k; - gzip_http_version 1.1; - gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; - - map \$http_x_forwarded_proto \$fastcgi_https { - default \$https; - http ''; - https on; - } - - log_format moodle_combined '\$remote_addr - \$upstream_http_x_moodleuser [\$time_local] ' - '"\$request" \$status \$body_bytes_sent ' - '"\$http_referer" "\$http_user_agent"'; - - - include /etc/nginx/conf.d/*.conf; - include /etc/nginx/sites-enabled/*; -} -EOF - - # Configure nginx/https - cat <> /etc/nginx/sites-enabled/${siteFQDN}.conf -server { - listen 443 ssl; - root /moodle/html/moodle; - index index.php index.html index.htm; - - ssl on; - ssl_certificate /moodle/certs/nginx.crt; - ssl_certificate_key /moodle/certs/nginx.key; - - # Log to syslog - error_log syslog:server=localhost,facility=local1,severity=error,tag=moodle; - access_log syslog:server=localhost,facility=local1,severity=notice,tag=moodle moodle_combined; - - # Log XFF IP instead of varnish - set_real_ip_from 10.0.0.0/8; - set_real_ip_from 127.0.0.1; - set_real_ip_from 172.16.0.0/12; - set_real_ip_from 192.168.0.0/16; - real_ip_header X-Forwarded-For; - real_ip_recursive on; - - location / { - proxy_set_header Host \$host; - proxy_set_header HTTP_REFERER \$http_referer; - proxy_set_header X-Forwarded-Host \$host; - proxy_set_header X-Forwarded-Server \$host; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; - proxy_pass http://localhost:80; - - proxy_connect_timeout 3600; - proxy_send_timeout 3600; - proxy_read_timeout 3600; - send_timeout 3600; - } -} -EOF - - if [ "$webServerType" = "nginx" ]; then - cat <> /etc/nginx/sites-enabled/${siteFQDN}.conf -server { - listen 81 default; - server_name ${siteFQDN}; - root /moodle/html/moodle; - index index.php index.html index.htm; - - # Log to syslog - error_log syslog:server=localhost,facility=local1,severity=error,tag=moodle; - access_log syslog:server=localhost,facility=local1,severity=notice,tag=moodle moodle_combined; - - # Log XFF IP instead of varnish - set_real_ip_from 10.0.0.0/8; - set_real_ip_from 127.0.0.1; - set_real_ip_from 172.16.0.0/12; - set_real_ip_from 192.168.0.0/16; - real_ip_header X-Forwarded-For; - real_ip_recursive on; - - - # Redirect to https - if (\$http_x_forwarded_proto != https) { - return 301 https://\$server_name\$request_uri; - } - rewrite ^/(.*\.php)(/)(.*)$ /\$1?file=/\$3 last; - - - # Filter out php-fpm status page - location ~ ^/server-status { - return 404; - } - - location / { - try_files \$uri \$uri/index.php?\$query_string; - } - - location ~ [^/]\.php(/|$) { - fastcgi_split_path_info ^(.+?\.php)(/.*)$; - if (!-f \$document_root\$fastcgi_script_name) { - return 404; - } + #NGINX / PHP config + create_nginx_configuration ${siteFQDN} - fastcgi_buffers 16 16k; - fastcgi_buffer_size 32k; - fastcgi_param SCRIPT_FILENAME \$document_root\$fastcgi_script_name; - fastcgi_pass unix:/run/php/php7.0-fpm.sock; - fastcgi_read_timeout 3600; - fastcgi_index index.php; - include fastcgi_params; - } -} - -EOF - fi - - if [ "$webServerType" = "apache" ]; then - # Configure Apache/php - sed -i "s/Listen 80/Listen 81/" /etc/apache2/ports.conf - a2enmod rewrite && a2enmod remoteip && a2enmod headers - - cat <> /etc/apache2/sites-enabled/${siteFQDN}.conf - - ServerName ${siteFQDN} - - ServerAdmin webmaster@localhost - DocumentRoot /moodle/html/moodle - - - Options FollowSymLinks - AllowOverride All - Require all granted - - - # Redirect unencrypted direct connections to HTTPS - - RewriteEngine on - RewriteCond %{HTTP:X-Forwarded-Proto} !https [NC] - RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [L,R=301] - - - # Log X-Forwarded-For IP address instead of varnish (127.0.0.1) - SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded - LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined - LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" forwarded - ErrorLog "|/usr/bin/logger -t moodle -p local1.error" - CustomLog "|/usr/bin/logger -t moodle -p local1.notice" combined env=!forwarded - CustomLog "|/usr/bin/logger -t moodle -p local1.notice" forwarded env=forwarded - - -EOF - fi - - # php config - if [ "$webServerType" = "apache" ]; then - PhpIni=/etc/php/7.0/apache2/php.ini - else - PhpIni=/etc/php/7.0/fpm/php.ini - fi - sed -i "s/memory_limit.*/memory_limit = 512M/" $PhpIni - sed -i "s/max_execution_time.*/max_execution_time = 18000/" $PhpIni - sed -i "s/max_input_vars.*/max_input_vars = 100000/" $PhpIni - sed -i "s/max_input_time.*/max_input_time = 600/" $PhpIni - sed -i "s/upload_max_filesize.*/upload_max_filesize = 1024M/" $PhpIni - sed -i "s/post_max_size.*/post_max_size = 1056M/" $PhpIni - sed -i "s/;opcache.use_cwd.*/opcache.use_cwd = 1/" $PhpIni - sed -i "s/;opcache.validate_timestamps.*/opcache.validate_timestamps = 1/" $PhpIni - sed -i "s/;opcache.save_comments.*/opcache.save_comments = 1/" $PhpIni - sed -i "s/;opcache.enable_file_override.*/opcache.enable_file_override = 0/" $PhpIni - sed -i "s/;opcache.enable.*/opcache.enable = 1/" $PhpIni - sed -i "s/;opcache.memory_consumption.*/opcache.memory_consumption = 256/" $PhpIni - sed -i "s/;opcache.max_accelerated_files.*/opcache.max_accelerated_files = 8000/" $PhpIni - - # Remove the default site. Moodle is the only site we want - rm -f /etc/nginx/sites-enabled/default - if [ "$webServerType" = "apache" ]; then - rm -f /etc/apache2/sites-enabled/000-default.conf - fi - - # restart Nginx - sudo service nginx restart - - if [ "$webServerType" = "nginx" ]; then - # fpm config - overload this - cat < /etc/php/7.0/fpm/pool.d/www.conf -[www] -user = www-data -group = www-data -listen = /run/php/php7.0-fpm.sock -listen.owner = www-data -listen.group = www-data -pm = dynamic -pm.max_children = 3000 -pm.start_servers = 20 -pm.min_spare_servers = 20 -pm.max_spare_servers = 30 -EOF - - # Restart fpm - service php7.0-fpm restart - fi - - if [ "$webServerType" = "apache" ]; then - sudo service apache2 restart - fi # Configure varnish startup for 16.04 VARNISHSTART="ExecStart=\/usr\/sbin\/varnishd -j unix,user=vcache -F -a :80 -T localhost:6082 -f \/etc\/varnish\/moodle.vcl -S \/etc\/varnish\/secret -s malloc,1024m -p thread_pool_min=200 -p thread_pool_max=4000 -p thread_pool_add_delay=2 -p timeout_linger=100 -p timeout_idle=30 -p send_timeout=1800 -p thread_pools=4 -p http_max_hdr=512 -p workspace_backend=512k" sed -i "s/^ExecStart.*/${VARNISHSTART}/" /lib/systemd/system/varnish.service - # Configure varnish VCL for moodle - cat <> /etc/varnish/moodle.vcl -vcl 4.0; - -import std; -import directors; -backend default { - .host = "localhost"; - .port = "81"; - .first_byte_timeout = 3600s; - .connect_timeout = 600s; - .between_bytes_timeout = 600s; -} - -sub vcl_recv { - # Varnish does not support SPDY or HTTP/2.0 untill we upgrade to Varnish 5.0 - if (req.method == "PRI") { - return (synth(405)); - } - - if (req.restarts == 0) { - if (req.http.X-Forwarded-For) { - set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip; - } else { - set req.http.X-Forwarded-For = client.ip; - } - } - - # Non-RFC2616 or CONNECT HTTP requests methods filtered. Pipe requests directly to backend - if (req.method != "GET" && - req.method != "HEAD" && - req.method != "PUT" && - req.method != "POST" && - req.method != "TRACE" && - req.method != "OPTIONS" && - req.method != "DELETE") { - return (pipe); - } - - # Varnish don't mess with healthchecks - if (req.url ~ "^/admin/tool/heartbeat" || req.url ~ "^/healthcheck.php") - { - return (pass); - } - - # Pipe requests to backup.php straight to backend - prevents problem with progress bar long polling 503 problem - # This is here because backup.php is POSTing to itself - Filter before !GET&&!HEAD - if (req.url ~ "^/backup/backup.php") - { - return (pipe); - } - - # Varnish only deals with GET and HEAD by default. If request method is not GET or HEAD, pass request to backend - if (req.method != "GET" && req.method != "HEAD") { - return (pass); - } - - ### Rules for Moodle and Totara sites ### - # Moodle doesn't require Cookie to serve following assets. Remove Cookie header from request, so it will be looked up. - if ( req.url ~ "^/altlogin/.+/.+\.(png|jpg|jpeg|gif|css|js|webp)$" || - req.url ~ "^/pix/.+\.(png|jpg|jpeg|gif)$" || - req.url ~ "^/theme/font.php" || - req.url ~ "^/theme/image.php" || - req.url ~ "^/theme/javascript.php" || - req.url ~ "^/theme/jquery.php" || - req.url ~ "^/theme/styles.php" || - req.url ~ "^/theme/yui" || - req.url ~ "^/lib/javascript.php/-1/" || - req.url ~ "^/lib/requirejs.php/-1/" - ) - { - set req.http.X-Long-TTL = "86400"; - unset req.http.Cookie; - return(hash); - } - - # Perform lookup for selected assets that we know are static but Moodle still needs a Cookie - if( req.url ~ "^/theme/.+\.(png|jpg|jpeg|gif|css|js|webp)" || - req.url ~ "^/lib/.+\.(png|jpg|jpeg|gif|css|js|webp)" || - req.url ~ "^/pluginfile.php/[0-9]+/course/overviewfiles/.+\.(?i)(png|jpg)$" - ) - { - # Set internal temporary header, based on which we will do things in vcl_backend_response - set req.http.X-Long-TTL = "86400"; - return (hash); - } - - # Serve requests to SCORM checknet.txt from varnish. Have to remove get parameters. Response body always contains "1" - if ( req.url ~ "^/lib/yui/build/moodle-core-checknet/assets/checknet.txt" ) - { - set req.url = regsub(req.url, "(.*)\?.*", "\1"); - unset req.http.Cookie; # Will go to hash anyway at the end of vcl_recv - set req.http.X-Long-TTL = "86400"; - return(hash); - } - - # Requests containing "Cookie" or "Authorization" headers will not be cached - if (req.http.Authorization || req.http.Cookie) { - return (pass); - } - - # Almost everything in Moodle correctly serves Cache-Control headers, if - # needed, which varnish will honor, but there are some which don't. Rather - # than explicitly finding them all and listing them here we just fail safe - # and don't cache unknown urls that get this far. - return (pass); -} - -sub vcl_backend_response { - # Happens after we have read the response headers from the backend. - # - # Here you clean the response headers, removing silly Set-Cookie headers - # and other mistakes your backend does. - - # We know these assest are static, let's set TTL >0 and allow client caching - if ( beresp.http.Cache-Control && bereq.http.X-Long-TTL && beresp.ttl < std.duration(bereq.http.X-Long-TTL + "s", 1s) && !beresp.http.WWW-Authenticate ) - { # If max-age < defined in X-Long-TTL header - set beresp.http.X-Orig-Pragma = beresp.http.Pragma; unset beresp.http.Pragma; - set beresp.http.X-Orig-Cache-Control = beresp.http.Cache-Control; - set beresp.http.Cache-Control = "public, max-age="+bereq.http.X-Long-TTL+", no-transform"; - set beresp.ttl = std.duration(bereq.http.X-Long-TTL + "s", 1s); - unset bereq.http.X-Long-TTL; - } - else if( !beresp.http.Cache-Control && bereq.http.X-Long-TTL && !beresp.http.WWW-Authenticate ) { - set beresp.http.X-Orig-Pragma = beresp.http.Pragma; unset beresp.http.Pragma; - set beresp.http.Cache-Control = "public, max-age="+bereq.http.X-Long-TTL+", no-transform"; - set beresp.ttl = std.duration(bereq.http.X-Long-TTL + "s", 1s); - unset bereq.http.X-Long-TTL; - } - else { # Don't touch headers if max-age > defined in X-Long-TTL header - unset bereq.http.X-Long-TTL; - } - - # Here we set X-Trace header, prepending it to X-Trace header received from backend. Useful for troubleshooting - if(beresp.http.x-trace && !beresp.was_304) { - set beresp.http.X-Trace = regsub(server.identity, "^([^.]+),?.*$", "\1")+"->"+regsub(beresp.backend.name, "^(.+)\((?:[0-9]{1,3}\.){3}([0-9]{1,3})\)","\1(\2)")+"->"+beresp.http.X-Trace; - } - else { - set beresp.http.X-Trace = regsub(server.identity, "^([^.]+),?.*$", "\1")+"->"+regsub(beresp.backend.name, "^(.+)\((?:[0-9]{1,3}\.){3}([0-9]{1,3})\)","\1(\2)"); - } - - # Gzip JS, CSS is done at the ngnix level doing it here dosen't respect the no buffer requsets - # if (beresp.http.content-type ~ "application/javascript.*" || beresp.http.content-type ~ "text") { - # set beresp.do_gzip = true; - #} -} - -sub vcl_deliver { - - # Revert back to original Cache-Control header before delivery to client - if (resp.http.X-Orig-Cache-Control) - { - set resp.http.Cache-Control = resp.http.X-Orig-Cache-Control; - unset resp.http.X-Orig-Cache-Control; - } - - # Revert back to original Pragma header before delivery to client - if (resp.http.X-Orig-Pragma) - { - set resp.http.Pragma = resp.http.X-Orig-Pragma; - unset resp.http.X-Orig-Pragma; - } - - # (Optional) X-Cache HTTP header will be added to responce, indicating whether object was retrieved from backend, or served from cache - if (obj.hits > 0) { - set resp.http.X-Cache = "HIT"; - } else { - set resp.http.X-Cache = "MISS"; - } - - # Set X-AuthOK header when totara/varnsih authentication succeeded - if (req.http.X-AuthOK) { - set resp.http.X-AuthOK = req.http.X-AuthOK; - } - - # If desired "Via: 1.1 Varnish-v4" response header can be removed from response - unset resp.http.Via; - unset resp.http.Server; - - return(deliver); -} - -sub vcl_backend_error { - # More comprehensive varnish error page. Display time, instance hostname, host header, url for easier troubleshooting. - set beresp.http.Content-Type = "text/html; charset=utf-8"; - set beresp.http.Retry-After = "5"; - synthetic( {" - - - - "} + beresp.status + " " + beresp.reason + {" - - -

Error "} + beresp.status + " " + beresp.reason + {"

-

"} + beresp.reason + {"

-

Guru Meditation:

-

Time: "} + now + {"

-

Node: "} + server.hostname + {"

-

Host: "} + bereq.http.host + {"

-

URL: "} + bereq.url + {"

-

XID: "} + bereq.xid + {"

-
-

Varnish cache server - - - "} ); - return (deliver); -} - -sub vcl_synth { - - #Redirect using '301 - Permanent Redirect', permanent redirect - if (resp.status == 851) { - set resp.http.Location = req.http.x-redir; - set resp.http.X-Varnish-Redirect = true; - set resp.status = 301; - return (deliver); - } - - #Redirect using '302 - Found', temporary redirect - if (resp.status == 852) { - set resp.http.Location = req.http.x-redir; - set resp.http.X-Varnish-Redirect = true; - set resp.status = 302; - return (deliver); - } - - #Redirect using '307 - Temporary Redirect', !GET&&!HEAD requests, dont change method on redirected requests - if (resp.status == 857) { - set resp.http.Location = req.http.x-redir; - set resp.http.X-Varnish-Redirect = true; - set resp.status = 307; - return (deliver); - } - - #Respond with 403 - Forbidden - if (resp.status == 863) { - set resp.http.X-Varnish-Error = true; - set resp.status = 403; - return (deliver); - } -} -EOF - - # Restart Varnish - systemctl daemon-reload - service varnish restart - + # Configure varnish VCL for moodle + create_varnish_configuration } > /tmp/setup.log