Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
tree: 3eacb63bcc
Fetching contributors…

Cannot retrieve contributors at this time

529 lines (445 sloc) 14.912 kb
#!/bin/sh
#
# This is a script used by Norway to bootstrap/update their FixMyStreet and
# mapit installations. It may be helpful to you in order to see what it does,
# but also is quite specific about where things are located and so on.
# We hope to improve the installation process in the near future.
#
# Single argument is one of 'bootstrap' 'update' 'remove' or 'update_map'
# Exit on first error
set -e
PATH=/sbin:/usr/sbin:$PATH
export PATH
servername=$(hostname)
basedir=/srv/$servername
codename=$(lsb_release -cs)
install_debs() {
# Statistics::Distributions is created locally
dpkg -i /root/libstatistics-distributions-perl_1.02-1_all.deb
# Install fixmystreet dependencies
apt-get install -y $(cat fixmystreet/conf/packages) \
libregexp-common-perl
# libcache-memcached-perl libfile-slurp-perl \
# libgeography-nationalgrid-perl liberror-perl libwww-perl \
# libimage-magick-perl libdbd-pg-perl libio-string-perl \
# libregexp-common-perl libmath-bigint-gmp-perl libdigest-sha1-perl \
# libxml-rss-perl libfcgi-perl libjson-perl libimage-size-perl \
# php5-cli postgresql apache2 libtext-template-perl libtext-csv-perl
# Install mapit dependencies
apt-get install -y $(cat mapit/conf/packages)
apt-get install -y libapache2-mod-fcgid
apt-get install -y pwgen
apt-get install -y gnuplot ttf-bitstream-vera
# XXX Also need to fetch and compile
# https://secure.mysociety.org/cvstrac/dir?d=mysociety/run-with-lockfile
}
# Load Debian and CPAN packages needed for the Catalyst version
install_dev_catalyst() {
apt-get install -y libcatalyst-perl \
libreadonly-perl \
libclass-c3-componentised-perl \
libfile-find-rule-perl \
libcatalyst-modules-perl \
libcatalyst-modules-extra-perl \
libdatetime-format-iso8601-perl \
libdatetime-format-http-perl \
libdatetime-format-pg-perl \
libdbd-pg-perl \
libsub-override-perl \
libhtml-treebuilder-xpath-perl \
libtest-requires-perl \
libhtml-selector-xpath-perl \
libtest-base-perl \
libdbix-class-encodedcolumn-perl \
libcrypt-eksblowfish-perl \
libdatetime-format-w3cdtf-perl \
libhaml-ruby
# Workaround for missing Perl modules in Debian
(
installcpandeb() {
pkg=$1
echo "info: Generating deb for $pkg"
debs=$(LC_ALL=C cpan2deb $pkg 2>&1 |grep 'dpkg-deb: building package'| sed "s/.* in \`\(.*\)'./\1/" | sed 's%^../%%')
echo "info: Installing $debs"
dpkg -i $debs
}
mkdir -p /srv/src
cd /srv/src
apt-get install -y dh-make-perl apt-file
apt-file update
installcpandeb URI::SmartURI
installcpandeb Catalyst::Plugin::SmartURI
installcpandeb Catalyst::Plugin::Unicode::Encoding
installcpandeb Web::Scraper
installcpandeb Sort::Key
installcpandeb Term::Size::Perl
installcpandeb Devel::Hide
installcpandeb Term::Size::Any
installcpandeb Template::Plugin::DateTime::Format
installcpandeb Template::Plugin::Comma
# Upgrade from 0.34 in Squeeze, to get expose_methods provided
# since 0.35.
installcpandeb Catalyst::View::TT
# App::cpanminus, Bundle::DBD::Pg, CPAN::ParseDistribution
)
}
pgsql_createuser() {
dbuser="$1"
dbpassword="$2"
su postgres -c "createuser -SDRl $dbuser"
su postgres -c "psql -c \"alter user \\\"$dbuser\\\" with password '$dbpassword';\""
}
pgsql_fixmystreet_bootstrap() {
dbuser="$1"
dbname="$2"
su postgres -c "createdb -E UTF8 --owner $dbuser $dbname; createlang plpgsql $dbname"
su $dbuser -c "psql $dbname -f $basedir/fixmystreet/db/schema.sql"
su $dbuser -c "psql $dbname -f $basedir/fixmystreet/db/alert_types.sql"
}
pgsql_mapit_bootstrap() {
dbuser="$1"
dbname="$2"
su postgres -c "createdb -E UTF8 -T template_postgis --owner $dbuser $dbname"
# No need to use "createlang plpgsql $dbname", as template_postgis
# already include it.
}
pgsql_remove_db() {
dbname="$1"
su postgres -c "dropdb $dbname"
}
pgsql_remove_user() {
dbuser="$1"
su postgres -c "dropuser $dbuser"
}
postgis_bootstrap() {
case "$codename" in
lenny)
POSTGIS_SQL_PATH=/usr/share/postgresql-8.3-postgis
POSTGISSQL=lwpostgis.sql
;;
*)
POSTGIS_SQL_PATH=/usr/share/postgresql/8.4/contrib/postgis-1.5
POSTGISSQL=postgis.sql
;;
esac
su postgres -c "createdb -E UTF8 template_postgis"
su postgres -c "createlang -d template_postgis plpgsql"
su postgres -c "psql -d postgres -c \"UPDATE pg_database SET datistemplate='true' WHERE datname='template_postgis';\""
# Loading the PostGIS SQL routines
su postgres -c "psql -d template_postgis -f $POSTGIS_SQL_PATH/$POSTGISSQL"
su postgres -c "psql -d template_postgis -f $POSTGIS_SQL_PATH/spatial_ref_sys.sql"
# Enabling users to alter spatial tables.
su postgres -c "psql -d template_postgis -c \"GRANT ALL ON geometry_columns TO PUBLIC;\""
su postgres -c "psql -d template_postgis -c \"GRANT ALL ON spatial_ref_sys TO PUBLIC;\""
}
postgis_remove() {
# Change template status to make it possible to drop the database
su postgres -c "psql -d postgres -c \"UPDATE pg_database SET datistemplate='false' WHERE datname='template_postgis';\""
su postgres -c "dropdb template_postgis"
}
apache_config() {
cat > /etc/apache2/sites-available/fixmystreet <<EOF
<VirtualHost *:80>
ServerName $servername
ServerAlias fiksgatami.no www.fiksgatami.no
DocumentRoot $basedir/fixmystreet/web/
# 5 MB limit to allow large images to be uploaded
FcgidMaxRequestLen 5242880
RewriteEngine on
RewriteCond %{HTTP_HOST} ^fiksgatami\.no
RewriteRule ^(.*)$ http://www.fiksgatami.no$1 [R=permanent,L]
RewriteCond %{HTTP_HOST} ^fiksgatami\.nuug\.no
RewriteRule ^(.*)$ http://www.fiksgatami.no$1 [R=permanent,L]
# Pull in the specific config
Include $basedir/fixmystreet/conf/httpd.conf
<Directory $basedir/fixmystreet/web>
# You also need to enable cgi files to run as CGI scripts. For example:
# on production servers these are run under fastcgi
Options +ExecCGI
AddHandler fcgid-script .cgi
</Directory>
<Location /admin>
Options +ExecCGI
AddHandler fcgid-script .cgi
AllowOverride AuthConfig
Order Allow,Deny
Allow From All
AuthName "Fiksgatami Admin Access"
AuthType Basic
AuthUserFile /etc/apache2/htpasswd.fixmystreet.users
require valid-user
</Location>
Alias /admin/ $basedir/fixmystreet/web-admin/
Alias /jslib/ $basedir/fixmystreet/commonlib/jslib/
</VirtualHost>
EOF
(
cd $basedir
sed s/mapit.mysociety.org/mapit.nuug.no/ \
< fixmystreet/conf/httpd.conf-example \
> fixmystreet/conf/httpd.conf
)
htpwdfile=/etc/apache2/htpasswd.fixmystreet.users
if [ ! -e $htpwdfile ] ; then
htpasswd -cmb $htpwdfile admin "$webpassword"
fi
case "$codename" in
lenny)
djangodir=/var/lib/python-support/python2.5/django
;;
*)
djangodir=/var/lib/python-support/python2.6/django
;;
esac
cat > /etc/apache2/sites-available/mapit <<EOF
<VirtualHost *:80>
ServerName mapit.nuug.no
ServerAlias mapit-dev.nuug.no
DocumentRoot $basedir/mapit/web/
WSGIDaemonProcess mapit.nuug.no user=www-data group=www-data processes=5 threads=1 display-name=mapit.nuug.no
WSGIProcessGroup mapit.nuug.no
WSGIScriptAlias / $basedir/mapit/web/django.wsgi
Alias /media $djangodir/contrib/admin/media
Alias /static $basedir/mapit/mapit/static
<Directory $basedir/mapit/web>
# You also need to enable cgi files to run as CGI scripts. For example:
# on production servers these are run under fastcgi
Options +ExecCGI
AddHandler fcgid-script .cgi
</Directory>
</VirtualHost>
EOF
a2dissite default
a2ensite fixmystreet
a2ensite mapit
a2enmod proxy rewrite
a2enmod proxy_http
a2enmod expires
/etc/init.d/apache2 restart
}
apache_stop() {
/etc/init.d/apache2 stop
}
apache_remove() {
a2dissite fixmystreet
a2dissite mapit
/etc/init.d/apache2 restart
rm /etc/apache2/sites-available/fixmystreet
rm /etc/apache2/sites-available/mapit
}
fetch_git_source() {
for gitmodule in fixmystreet mapit ; do
if [ ! -d $gitmodule ] ; then
git clone git://github.com/petterreinholdtsen/$gitmodule.git -b $servername || \
git clone https://github.com/mysociety/$gitmodule.git
(cd $gitmodule &&
git submodule update --init)
else
(cd $gitmodule &&
git pull &&
git submodule update &&
git diff) | cat
fi
done
# Update translation files
(cd fixmystreet && commonlib/bin/gettext-makemo)
# Generate CSS files
(cd fixmystreet && bin/make_css)
# Not quite sure why this directory is needed
mkdir -p fixmystreet/perl-external/local-lib
}
update_fixmystreet_config() {
dbuser="$1"
fgmdbpassword="$2"
dbname="$3"
cat > $basedir/fixmystreet/conf/general.yml <<EOF
# general-example.yml:
# Example values for the "general" config file.
#
# Configuration parameters, in YAML syntax.
#
# Copy this file to one called "general.yml" in the same directory. Or
# have multiple config files and use a symlink to change between them.
FMS_DB_HOST: 'localhost'
FMS_DB_PORT: '5432'
FMS_DB_NAME: '$dbname'
FMS_DB_USER: '$dbuser'
FMS_DB_PASS: '$fgmdbpassword'
BASE_URL: 'http://$servername'
EMAIL_DOMAIN: 'nuug.no'
CONTACT_EMAIL: ''
CONTACT_NAME: 'FiksGataMi'
STAGING_SITE: 1
UPLOAD_DIR: '/var/lib/fixmystreet/upload/'
GEO_CACHE: '/var/lib/fixmystreet/cache/'
GOOGLE_MAPS_API_KEY: ''
MAPIT_URL: 'http://mapit.nuug.no'
MAP_TYPE: 'OSM'
GAZE_URL: 'http://gaze.mysociety.org/gaze'
# empty = use /usr/sbin/sendmail'
SMTP_SMARTHOST: ''
AUTH_SHARED_SECRET: ''
HEARFROMYOURMP_BASE_URL: ''
ANDROID_URL: 'http://wiki.nuug.no/grupper/fiksgatami/android'
ALLOWED_COBRANDS: 'fiksgatami'
# How many items are returned in the GeoRSS feeds by default
RSS_LIMIT: '200'
OSM_GUESS_OPERATOR: 1
# Should problem reports link to the council summary pages?
AREA_LINKS_FROM_PROBLEMS: 1
EOF
if [ ! -d /var/lib/fixmystreet ] ; then
(
mkdir /var/lib/fixmystreet && \
cd /var/lib/fixmystreet && \
mkdir cache && \
mkdir upload && \
chown www-data cache && \
chown www-data upload
)
fi
}
update_mapit_config() {
dbuser="$1"
dbpassword="$2"
dbname="$3"
if [ -e $basedir/mapit/conf/general ] ; then
mv $basedir/mapit/conf/general $basedir/mapit/conf/general.old-$$
fi
cat > $basedir/mapit/conf/general.yml <<EOF
# Connection details for database
MAPIT_DB_NAME: '$dbname'
MAPIT_DB_USER: '$dbuser'
MAPIT_DB_PASS: '$dbpassword'
MAPIT_DB_HOST: null
MAPIT_DB_PORT: null
# Country is currently one of GB, NO, or KE
COUNTRY: 'NO'
# An EPSG code for what the areas are stored as. 27700 is OSGB, 4326 for WGS84.
AREA_SRID: 4326
STAGING: 0
RATE_LIMIT: []
BUGS_EMAIL: ''
DJANGO_SECRET_KEY: ''
EOF
}
mapit_manage="$basedir/mapit/project/manage.py"
load_mapit_n5000() {
N5000SHP="$basedir/N5000 shape/N5000_AdministrativFlate.shp"
su www-data -c "$mapit_manage generation_create --commit --desc 'N5000 Norway'"
su www-data -c "$mapit_manage import_norway_n5000 --commit '$N5000SHP'"
# su www-data -c "$mapit_manage find_parents"
su www-data -c "$mapit_manage generation_activate --commit"
}
load_mapit_osm() {
python $basedir/mapit/bin/osm_to_kml
# This require changes in the code
if [ new-generation = "$1" ] ; then
su www-data -c "$mapit_manage generation_create --commit --desc 'OpenStreetmap.org'"
fi
su www-data -c "$mapit_manage import_norway_osm --commit $basedir/mapit/data/cache/*.kml"
# su www-data -c "$mapit_manage find_parents"
# su www-data -c "$mapit_manage import_area_unions --commit data/norway/regions.csv"
if [ new-generation = "$1" ] ; then
su www-data -c "$mapit_manage generation_activate --commit"
fi
}
load_mapit_postcodes() {
rm -f postnummer-utf8.txt postnummer-ekstra.csv
wget http://www.erikbolstad.no/nedlasting/postnummer-utf8.txt
wget http://www.nuug.no/prosjekt/fiksgatami/postnummer-ekstra.csv
su www-data -c "$mapit_manage import_bolstad_postcodes postnummer-utf8.txt"
su www-data -c "$mapit_manage import_bolstad_postcodes postnummer-ekstra.csv"
}
# This one need to run as user www-data
load_mapit() {
su www-data -c "$mapit_manage syncdb --noinput"
su www-data -c "$mapit_manage migrate"
su www-data -c "$mapit_manage loaddata norway"
#load_mapit_n5000
load_mapit_osm new-generation
load_mapit_postcodes || true
}
setup_locale() {
cat > /etc/locale.gen <<EOF
en_GB.UTF-8 UTF-8
en_US.UTF-8 UTF-8
nb_NO.UTF-8 UTF-8
nn_NO.UTF-8 UTF-8
EOF
locale-gen
}
flush_memcache() {
(echo flush_all; sleep 1) | telnet localhost 11211 || true
}
append_if_missing() {
file="$1"
string="$2"
if [ -e "$file" ] ; then
if ! grep -qxF "$string" "$file" ; then
echo "Appending '$string' to $file."
echo "$string" >> $file
fi
fi
}
tune_postgresql() {
conf=/etc/postgresql/8.4/main/postgresql.conf
append_if_missing $conf "effective_cache_size = 256MB"
append_if_missing $conf "shared_buffers = 128MB"
append_if_missing $conf "log_min_duration_statement = 250"
touch /etc/sysctl.d/postgresql.conf
append_if_missing /etc/sysctl.d/postgresql.conf "kernel.shmmax = 136314880"
sysctl -w kernel.shmmax=$(( 130 * 1024 * 1024 ))
/etc/init.d/postgresql reload
}
cd /srv/$servername/
case "$1" in
bootstrap)
fetch_git_source
install_debs
if [ yes = "$CATALYST" ] ; then
install_dev_catalyst
fi
# Not yet ready, should adjust dynamically based on memory size
# tune_postgresql
# Password to use when connecting to the postgresql database.
fgmdbpassword="$(pwgen -1)"
midbpassword="$(pwgen -1)"
webpassword="$(pwgen -1)"
postgis_bootstrap
pgsql_createuser www-data "$fgmdbpassword"
pgsql_createuser mapit "$midbpassword"
pgsql_fixmystreet_bootstrap www-data fixmystreet
update_fixmystreet_config www-data "$fgmdbpassword" fixmystreet
pgsql_mapit_bootstrap www-data mapit
update_mapit_config www-data "$midbpassword" mapit
load_mapit
setup_locale
apache_config
;;
remove)
apache_stop
flush_memcache
pgsql_remove_db fixmystreet || true
pgsql_remove_db mapit || true
pgsql_remove_user mapit || true
pgsql_remove_user www-data || true
postgis_remove || true
apache_remove || true
rm $basedir/mapit/data/cache/relation-*
;;
update)
fetch_git_source
fgmdbpassword=$(grep OPTION_FMS_DB_PASS $basedir/fixmystreet/conf/general | cut -d\' -f4)
midbpassword=$(grep OPTION_MAPIT_DB_PASS $basedir/mapit/conf/general | cut -d\' -f4)
update_fixmystreet_config www-data $fgmdbpassword fixmystreet
update_mapit_config www-data $midbpassword mapit
apache_config
flush_memcache
;;
update_map)
rm $basedir/mapit/data/cache/relation-*
load_mapit_osm
flush_memcache
;;
esac
Jump to Line
Something went wrong with that request. Please try again.