Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

cleanup

  • Loading branch information...
commit cd8aacdce6daf5c0be10434dc3304f11711e0622 1 parent c84429a
@emiltin emiltin authored
View
4 README.md
@@ -1,4 +1,4 @@
-osrm_setup
+ibikecph setup
===================
-Setup for the ibikecph.dk route server. Please note that these files work with a specific version of OSRM, currently our own custom branch at https://github.com/ibikecph/Project-OSRM/tree/ibikecph_alpha.
+Various scripts and configurations used on the ibikecph.dk servers for updating OSM and OSRM data, and regenerating Mapnik tiles.
View
2  contractor.ini
@@ -1,2 +0,0 @@
-Threads = 4
-SRTM = /opt/storage/srtm/Eurasia
View
1  extractor.ini
@@ -1 +0,0 @@
-Memory = 2
View
0  osrm.conf → routes_ibikecph_dk/osrm.conf
File renamed without changes
View
119 routes_ibikecph_dk/update
@@ -0,0 +1,119 @@
+#!/usr/bin/ruby
+
+require 'rubygems'
+require 'savon'
+
+USERNAME = 'emil.tin@tmf.kk.dk/6d3cb6b9-36c9-3337-b235-7f942744edcc'
+PASSWORD = '3djvxGE3xsd23a'
+SERVER_ID = 5373
+SERVER_USER = 'osm'
+SERVER_DOMAIN = 'tech.ibikecph.dk'
+SERVER_ROOT = '/home/osm'
+SERVER_COMMAND = 'update osm osrm shutdown' #skipping tiles for now. to run the full update cycle, simply use 'update'
+
+Savon.configure do |config|
+ config.log = false
+end
+
+HTTPI.log = false
+
+
+class UpdateScheduler
+
+ def initialize
+ @client = Savon::Client.new do
+ wsdl.document = "https://api.cloud.dk/?wsdl"
+ http.auth.basic USERNAME, PASSWORD
+ http.auth.ssl.verify_mode = :none
+ end
+ end
+
+ def get_status
+ response = @client.request :get_server, :body => { 'server_id' => SERVER_ID }
+ response.body[:get_server_response][:get_server_return][:status]
+ end
+
+ def wait_for_ssh
+ 600.times do
+ return true if system %{ssh #{SERVER_USER}@#{SERVER_DOMAIN} "whoami" 2>&1 > /dev/null}
+ sleep 1
+ end
+ raise "Timeout while waiting for SSH"
+ end
+
+ def launch
+ puts 'Launching server'
+ @client.request :start_server, :body => { 'server_id' => SERVER_ID }
+ end
+
+ def ensure_up
+ if get_status=='5' #stopped?
+ launch
+ else
+ puts 'Server already running'
+ end
+ puts 'Waiting for SSH'
+ wait_for_ssh
+ puts 'Server ready'
+ end
+
+ def shutdown
+ puts 'Shutting down server'
+ @client.request :stop_server, :body => { 'server_id' => SERVER_ID, 'stop_method' => 1 }
+ puts 'Server was shut down'
+ end
+
+
+
+ def down
+ puts "--------"
+ puts "Shutting down update server at #{Time.now}" # 1=graceful shutdown, 2=kill
+ @client.request :stop_server, :body => { 'server_id' => SERVER_ID, 'stop_method' => 1 }
+ end
+
+ def up
+ puts "--------"
+ puts "Launching update server at #{Time.now}"
+ ensure_up
+ end
+
+ def status
+ puts get_status
+ end
+
+ def update
+ puts "--------"
+ puts "Starting update at #{Time.now}"
+ ensure_up
+
+ #run script in background using '&'
+ #use nohup, so it's not terminated when we log out of ssh
+ #to avoid ssh hanging, make sure to redirect all three stream: stdout, stderr, stdin
+ puts "Initiating remote update at #{Time.now}... "
+ if system %{ssh #{SERVER_USER}@#{SERVER_DOMAIN} "nohup #{SERVER_ROOT}/#{SERVER_COMMAND} >> #{SERVER_ROOT}/update.log 2>&1 < /dev/null &"}
+ puts 'OK'
+ #we're done, remote script will handle shutdown after it finishes
+ else
+ raise '*** Failed to initiate remote update!'
+ end
+ rescue Exception => e
+ puts e
+ #if something goes wrong, ensure shutdown
+ shutdown
+ ensure
+ puts "\n\n"
+ end
+end
+
+
+scheduler = UpdateScheduler.new
+
+if ARGV[0]==nil
+ scheduler.update
+elsif ARGV[0]=='down'
+ scheduler.down
+elsif ARGV[0]=='up'
+ scheduler.up
+elsif ARGV[0]=='status'
+ scheduler.status
+end
View
10 server.ini
@@ -1,10 +0,0 @@
- Threads = 1
- IP = 83.221.133.2
- Port = 80
-
- hsgrData=osm_data/combined.osrm.hsgr
- nodesData=osm_data/combined.osrm.nodes
- edgesData=osm_data/combined.osrm.edges
- ramIndex=osm_data/combined.osrm.ramIndex
- fileIndex=osm_data/combined.osrm.fileIndex
- namesData=osm_data/combined.osrm.names
View
32 speedprofile.ini
@@ -1,32 +0,0 @@
-[bicycle]
- accessTag = bicycle
- defaultSpeed = 18
- obeyOneways = yes
- useRestrictions = yes
- obeyBollards = no
- turnPenalty = 60
- turnBias = 1.4
- trafficSignalPenalty = 12
- takeMinimumOfSpeeds = yes
- accessRestrictionKeys = destination
-
- cycleway = 18
- primary = 17
- primary_link = 17
- secondary = 18
- secondary_link = 18
- tertiary = 18
- residential = 18
- unclassified = 16
- living_street = 16
- road = 16
- service = 16
- track = 13
- path = 13
- footway = 5
- pedestrian = 5
- pier = 5
- steps = 1
-
-# excludeFromGrid = ferry
-# ferry = 5
View
142 tech_ibikecph_dk/update
@@ -0,0 +1,142 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+
+ROOT = File.expand_path '~'
+BIN_PATH = ROOT+'/Project-OSRM'
+DATA_FOLDER = ROOT+'/var'
+REPLICATION_FOLDER = 'replicate'
+PROFILE = ROOT+'/Project-OSRM/profiles/bicycle.lua'
+POLYGON = ROOT+'/var/map.poly'
+NAME = 'map'
+FILE = "#{NAME}.osm.pbf"
+NEWFILE = "#{NAME}_new.osm.pbf"
+USER = 'root'
+SERVER = 'routes.ibikecph.dk'
+SERVER_ROOT = '/home/osm'
+RENDER_OPTIONS = 'z=9-17 bbox=7.64,54.33,13.37,57.98 map=solar'
+SERVER_BIN_PATH = SERVER_ROOT + '/Project-OSRM'
+
+
+def time str, &block
+ start = Time.now
+ puts '---------'
+ puts "#{str}: Starting at #{start}"
+ yield block
+ finish = Time.now
+ seconds = (Time.now - start).to_i
+ formatted = format_time seconds
+ puts "#{str}: Completed in #{seconds}s / #{formatted}, at #{finish}."
+end
+
+def run_cmd cmd
+ puts cmd
+ raise "Failed to run command: #{cmd}" unless system cmd
+end
+
+def replicate
+ Dir.chdir DATA_FOLDER do
+ run_cmd "osmosis --read-replication-interval workingDirectory=#{REPLICATION_FOLDER} --simplify-change --read-pbf file=#{FILE} --apply-change --bounding-polygon file=#{POLYGON} --write-pbf file=#{NEWFILE} omitmetadata=true"
+ FileUtils.mv NEWFILE, FILE
+ end
+end
+
+def process
+ Dir.chdir DATA_FOLDER do
+ run_cmd "#{BIN_PATH}/osrm-extract #{FILE} #{PROFILE}"
+ puts
+ run_cmd "#{BIN_PATH}/osrm-prepare #{NAME}.osrm #{NAME}.osrm.restrictions #{PROFILE}"
+ puts
+ end
+end
+
+def rsync
+ run_cmd "rsync #{DATA_FOLDER}/*.osrm* #{USER}@#{SERVER}:/tmp/#{NAME}/"
+end
+
+def postgres
+ run_cmd "osm2pgsql -d osm -U osm -c -C8000 --number-processes=4 #{DATA_FOLDER}/#{FILE}"
+end
+
+def remove_tiles
+ run_cmd "rm -rf /tiles/plain/*"
+end
+
+def render_tiles
+ run_cmd "tirex-batch #{RENDER_OPTIONS}"
+ raw = ''
+ (20*60).times do |i|
+ sleep 60
+ raw = `tirex-status --raw`
+ size = /"size" : (\d+)/.match(raw)[1].to_i
+ if i%30 == 0
+ puts "#{size} tiles left to render, at #{Time.now}"
+ end
+ return if size==0
+ end
+ raise "Rendering timed out! Last tirex-status raw: #{raw}"
+end
+
+def convert_tiles
+ run_cmd "#{ROOT}/meta2tile /tiles/meta /tiles/plain"
+end
+
+def sync_tiles
+ run_cmd "rsync -r --ignore-times /tiles/plain/ tiles.ibikecph.dk:/tiles/plain/"
+end
+
+def deploy
+ log_msg = "OSRM update deployed at #{Time.now}"
+ if ARGV[0]=='bins'
+ run_cmd %{ssh #{USER}@#{SERVER} "stop osrm; rm -rf #{SERVER_ROOT}/data_old; mv #{SERVER_ROOT}/data #{SERVER_ROOT}/data_old; mv /tmp/#{NAME} #{SERVER_ROOT}/data"}
+ run_cmd %{scp #{BIN_PATH}/osrm-* #{USER}@#{SERVER}:#{SERVER_BIN_PATH}/}
+ run_cmd %{ssh #{USER}@#{SERVER} "start osrm; echo '#{log_msg}' >> #{SERVER_ROOT}/deploy.log"}
+ else
+ run_cmd %{ssh #{USER}@#{SERVER} "stop osrm; rm -rf #{SERVER_ROOT}/data_old; mv #{SERVER_ROOT}/data #{SERVER_ROOT}/data_old; mv /tmp/#{NAME} #{SERVER_ROOT}/data; start osrm; echo '#{log_msg}' >> #{SERVER_ROOT}/deploy.log"}
+ end
+end
+
+def shutdown
+ run_cmd "#{ROOT}/shutdown"
+end
+
+def format_time total_seconds
+ seconds = total_seconds % 60
+ minutes = (total_seconds / 60) % 60
+ hours = total_seconds / (60 * 60)
+ format("%02d:%02d:%02d", hours, minutes, seconds)
+end
+
+def update
+ puts '---------------------------------------'
+ time("Updating") do
+ begin
+ run_cmd "df -h"
+ run_cmd "free -m"
+ if ARGV.empty? || ARGV.include?('osm')
+ time("Updating OSM data") { replicate }
+ end
+ if ARGV.empty? || ARGV.include?('osrm')
+ time("Preprocess OSRM data") { process }
+ time("Sync data to route server") { rsync }
+ time("Swap folders and restart OSRM") { deploy }
+ end
+ if ARGV.empty? || ARGV.include?('tiles')
+ time("Import to Postgres") { postgres }
+ time("Render meta-tiles") { render_tiles }
+ time("Remove old tiles") { remove_tiles }
+ time("Convert meta-tiles") { convert_tiles }
+ time("Sync tiles to tiles server") { sync_tiles }
+ end
+ rescue Exception => e
+ puts "*** An error occurred:"
+ puts e
+ ensure
+ if ARGV.empty? || ARGV.include?('shutdown')
+ time("Shutdown") { shutdown }
+ end
+ end
+ end
+end
+
+update
Please sign in to comment.
Something went wrong with that request. Please try again.