Added sidekiq_monitor, unit tests.

Gemfile

@@ -38,6 +38,7 @@ gem 'ffi-geos'
 gem 'sinatra', '>= 1.3.0', :require => nil
 gem 'sidekiq'
 gem 'sidekiq_status'
+gem 'sidekiq_monitor'
 
 # Use ActiveModel has_secure_password
 # gem 'bcrypt-ruby', '~> 3.0.0'

Gemfile.lock

@@ -25,6 +25,7 @@ GEM
       multi_json (~> 1.3)
       thread_safe (~> 0.1)
       tzinfo (~> 0.3.37)
+    ajax-datatables-rails (0.0.1)
     arel (4.0.0)
     atomic (1.1.14)
     builder (3.1.4)
@@ -48,6 +49,8 @@ GEM
     jbuilder (1.5.1)
       activesupport (>= 3.0.0)
       multi_json (>= 1.2.0)
+    jquery-datatables-rails (1.11.2)
+      jquery-rails
     jquery-rails (3.0.4)
       railties (>= 3.0, < 5.0)
       thor (>= 0.14, < 2.0)
@@ -98,12 +101,20 @@ GEM
       json
       redis (>= 3.0.4)
       redis-namespace (>= 1.3.1)
+    sidekiq_monitor (0.0.9)
+      ajax-datatables-rails
+      jquery-datatables-rails
+      sidekiq (>= 2.2.1)
+      slim
     sidekiq_status (1.0.5)
       sidekiq (>= 2.4, < 2.15)
     sinatra (1.4.3)
       rack (~> 1.4)
       rack-protection (~> 1.4)
       tilt (~> 1.3, >= 1.3.4)
+    slim (2.0.1)
+      temple (~> 0.6.6)
+      tilt (>= 1.3.3, < 2.1)
     sprockets (2.10.0)
       hike (~> 1.2)
       multi_json (~> 1.0)
@@ -114,6 +125,7 @@ GEM
       activesupport (>= 3.0)
       sprockets (~> 2.8)
     sqlite3 (1.3.8)
+    temple (0.6.6)
     thor (0.18.1)
     thread_safe (0.1.3)
       atomic
@@ -142,6 +154,7 @@ DEPENDENCIES
   sass-rails (~> 4.0.0)
   sdoc
   sidekiq
+  sidekiq_monitor
   sidekiq_status
   sinatra (>= 1.3.0)
   sqlite3

Rakefile

@@ -3,4 +3,10 @@
 
 require File.expand_path('../config/application', __FILE__)
 
+Rake::TaskManager.class_eval do
+  def remove_task(task_name)
+    @tasks.delete(task_name.to_s)
+  end
+end
+
 Owl::Application.load_tasks

app/controllers/admin_controller.rb

@@ -0,0 +1,15 @@
+class AdminController < ActionController::Base
+  protect_from_forgery
+
+  def spawn_workers
+    #post_params[]
+    #TilerWorker.perform_async(params[:changeset_id])
+    sql = 'SELECT DISTINCT changeset_id AS id FROM nodes LIMIT 100'
+
+    for row in ActiveRecord::Base.connection.execute(sql) do
+      TilerWorker.perform_async(row['id'].to_i)
+    end
+
+    redirect_to '/admin'
+  end
+end

app/controllers/application_controller.rb

@@ -4,10 +4,6 @@ class ApplicationController < ActionController::Base
   before_filter :cors_preflight_check
   after_filter :cors_set_access_control_headers
 
-  def test
-    TilerWorker.perform_async(123, 321)
-  end
-
   # For all responses in this controller, return the CORS access control headers.
 
   def cors_set_access_control_headers

app/workers/tiler_worker.rb

@@ -1,33 +1,26 @@
 class TilerWorker
-  include SidekiqStatus::Worker
+  include Sidekiq::Worker
+  #include SidekiqStatus::Worker
 
   sidekiq_options :retry => false
 
-  def perform(name, count)
-    zoom = 16
-    changeset_ids = []
-
-    conn = ActiveRecord::Base.connection.raw_connection()
-    #conn.set_error_verbosity(0)
-    tiler = Tiler::ChangesetTiler.new(conn)
-
-    changeset_ids.each_with_index do |changeset_id, count|
-      next if changeset_id == 0
+  def initialize
+    @conn = ActiveRecord::Base.connection.raw_connection()
+    @conn.set_error_verbosity(0)
+    @tiler = Tiler::ChangesetTiler.new(@conn)
+  end
 
-      # Print out some diagnostic information.
-      if count % 1000 == 0
-        GC.start
-        p GC::stat
-        p GC::Profiler.result
-        p GC::Profiler.total_time
-        GC::Profiler.report
-      end
+  def perform(changeset_id)
+    zoom = 16
+    before = Time.now
+    puts "Generating tiles for changeset #{changeset_id}..."
+    tile_count = @tiler.generate(zoom, changeset_id, {})
+    puts "Done, tile count: #{tile_count}"
+    puts "Changeset #{changeset_id} took #{Time.now - before}s"
+    { tiles: tile_count }
+  end
 
-      before = Time.now
-      puts "Generating tiles for changeset #{changeset_id}... (#{count})"
-      tile_count = tiler.generate(zoom, changeset_id, options)
-      puts "Done, tile count: #{tile_count}"
-      puts "Changeset #{changeset_id} took #{Time.now - before}s (#{count})"
-    end
+  def self.job_name(changeset_id)
+    changeset_id
   end
 end

config/application.rb

@@ -20,7 +20,7 @@ class Application < Rails::Application
     # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
     # config.i18n.default_locale = :de
 
-    config.active_record.schema_format = :sql
+    #config.active_record.schema_format = :sql
 
     config.autoload_paths << "#{Rails.root}/lib/tiler"
   end

config/routes.rb

@@ -24,7 +24,10 @@
   # Map API
   get 'api/0.1/kothic/:zoom/:x/:y.js' => 'map_api#kothic', :constraints => @xyz_constrains
 
-  get 'test' => 'application#test'
+  #get 'admin/:changeset_id' => 'application#test'
+  #get 'admin/:changeset_id' => 'application#test'
+  post 'admin/spawn_workers' => 'admin#spawn_workers'
 
   mount Sidekiq::Web => '/sidekiq'
+  mount Sidekiq::Monitor::Engine => '/sidekiqm'
 end

db/migrate/20130928181457_create_sidekiq_jobs.rb

@@ -0,0 +1,26 @@
+class CreateSidekiqJobs < ActiveRecord::Migration
+  def change
+    create_table :sidekiq_jobs do |t|
+      t.string :jid
+      t.string :queue
+      t.string :class_name
+      t.text :args
+      t.boolean :retry
+      t.datetime :enqueued_at
+      t.datetime :started_at
+      t.datetime :finished_at
+      t.string :status
+      t.string :name
+      t.text :result
+    end
+
+    add_index :sidekiq_jobs, :jid
+    add_index :sidekiq_jobs, :queue
+    add_index :sidekiq_jobs, :retry
+    add_index :sidekiq_jobs, :class_name
+    add_index :sidekiq_jobs, :enqueued_at
+    add_index :sidekiq_jobs, :started_at
+    add_index :sidekiq_jobs, :finished_at
+    add_index :sidekiq_jobs, :status
+  end
+end

db/schema.rb

@@ -11,9 +11,108 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 0) do
+ActiveRecord::Schema.define(version: 20130928181457) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
+  enable_extension "hstore"
+  enable_extension "postgis"
+
+# Could not dump table "changeset_tiles" because of following StandardError
+#   Unknown type 'change' for column 'changes'
+
+# Could not dump table "changesets" because of following StandardError
+#   Unknown type 'geometry' for column 'bbox'
+
+  create_table "nodes", id: false, force: true do |t|
+    t.integer  "id",           limit: 8, null: false
+    t.integer  "version",                null: false
+    t.integer  "rev",                    null: false
+    t.boolean  "visible",                null: false
+    t.boolean  "current",                null: false
+    t.integer  "user_id",                null: false
+    t.datetime "tstamp",                 null: false
+    t.integer  "changeset_id", limit: 8, null: false
+    t.hstore   "tags",                   null: false
+    t.integer  "geom",         limit: 0
+  end
+
+  add_index "nodes", ["changeset_id"], name: "idx_nodes_changeset_id", using: :btree
+  add_index "nodes", ["geom"], name: "idx_nodes_geom", where: "(visible AND current)", using: :gist
+  add_index "nodes", ["id"], name: "idx_nodes_node_id", using: :btree
+
+  create_table "relation_members", id: false, force: true do |t|
+    t.integer "relation_id", limit: 8, null: false
+    t.integer "version",     limit: 8, null: false
+    t.integer "member_id",   limit: 8, null: false
+    t.string  "member_type", limit: 1, null: false
+    t.text    "member_role",           null: false
+    t.integer "sequence_id",           null: false
+  end
+
+  create_table "relations", id: false, force: true do |t|
+    t.integer  "id",           limit: 8, null: false
+    t.integer  "version",                null: false
+    t.integer  "rev",                    null: false
+    t.boolean  "visible",                null: false
+    t.boolean  "current",                null: false
+    t.integer  "user_id",                null: false
+    t.datetime "tstamp",                 null: false
+    t.integer  "changeset_id", limit: 8, null: false
+    t.hstore   "tags",                   null: false
+  end
+
+  add_index "relations", ["changeset_id"], name: "idx_relations_changeset_id", using: :btree
+
+  create_table "sidekiq_jobs", force: true do |t|
+    t.string   "jid"
+    t.string   "queue"
+    t.string   "class_name"
+    t.text     "args"
+    t.boolean  "retry"
+    t.datetime "enqueued_at"
+    t.datetime "started_at"
+    t.datetime "finished_at"
+    t.string   "status"
+    t.string   "name"
+    t.text     "result"
+  end
+
+  add_index "sidekiq_jobs", ["class_name"], name: "index_sidekiq_jobs_on_class_name", using: :btree
+  add_index "sidekiq_jobs", ["enqueued_at"], name: "index_sidekiq_jobs_on_enqueued_at", using: :btree
+  add_index "sidekiq_jobs", ["finished_at"], name: "index_sidekiq_jobs_on_finished_at", using: :btree
+  add_index "sidekiq_jobs", ["jid"], name: "index_sidekiq_jobs_on_jid", using: :btree
+  add_index "sidekiq_jobs", ["queue"], name: "index_sidekiq_jobs_on_queue", using: :btree
+  add_index "sidekiq_jobs", ["retry"], name: "index_sidekiq_jobs_on_retry", using: :btree
+  add_index "sidekiq_jobs", ["started_at"], name: "index_sidekiq_jobs_on_started_at", using: :btree
+  add_index "sidekiq_jobs", ["status"], name: "index_sidekiq_jobs_on_status", using: :btree
+
+  create_table "spatial_ref_sys", id: false, force: true do |t|
+    t.integer "srid",                   null: false
+    t.string  "auth_name", limit: 256
+    t.integer "auth_srid"
+    t.string  "srtext",    limit: 2048
+    t.string  "proj4text", limit: 2048
+  end
+
+  create_table "users", id: false, force: true do |t|
+    t.integer "id",   null: false
+    t.text    "name", null: false
+  end
+
+  create_table "ways", id: false, force: true do |t|
+    t.integer  "id",           limit: 8, null: false
+    t.integer  "version",                null: false
+    t.boolean  "visible",                null: false
+    t.boolean  "current",                null: false
+    t.integer  "user_id",                null: false
+    t.datetime "tstamp",                 null: false
+    t.integer  "changeset_id", limit: 8, null: false
+    t.hstore   "tags",                   null: false
+    t.integer  "nodes",        limit: 8, null: false, array: true
+  end
+
+  add_index "ways", ["changeset_id"], name: "idx_ways_changeset_id", using: :btree
+  add_index "ways", ["nodes"], name: "idx_ways_nodes_id", using: :gin
 
 end

db/sql/owl_functions.sql

@@ -288,8 +288,8 @@ BEGIN
   CREATE TEMPORARY TABLE _tmp_result ON COMMIT DROP AS
   SELECT *
   FROM _tmp_changeset_nodes n
-  WHERE (n.el_action IN ('CREATE', 'DELETE') OR n.tags_changed OR n.geom_changed) AND
-    (OWL_RemoveTags(n.tags) != ''::hstore OR OWL_RemoveTags(n.prev_tags) != ''::hstore);
+  WHERE (n.el_action IN ('CREATE', 'DELETE') OR n.tags_changed OR n.geom_changed);
+  --AND (OWL_RemoveTags(n.tags) != ''::hstore OR OWL_RemoveTags(n.prev_tags) != ''::hstore);
 
   GET DIAGNOSTICS row_count = ROW_COUNT;
   RAISE NOTICE '% --   Nodes done (%)', clock_timestamp(), row_count;

db/sql/owl_indexes.sql

@@ -21,4 +21,4 @@ CREATE INDEX idx_changeset_tiles_xcyz ON changeset_tiles USING btree (zoom, chan
 --CREATE INDEX idx_ways_tstamp ON ways USING btree (tstamp);
 
 -- Used by the vector tiles API.
-CREATE INDEX idx_nodes_geom ON nodes USING gist (geom) WHERE visible AND current;
+--CREATE INDEX idx_nodes_geom ON nodes USING gist (geom) WHERE visible AND current;

db/sql/owl_schema.sql

@@ -53,9 +53,7 @@ CREATE TABLE changesets (
 CREATE TABLE nodes (
   id bigint NOT NULL,
   version int NOT NULL,
-  rev int NOT NULL,
   visible boolean NOT NULL,
-  current boolean NOT NULL,
   user_id int NOT NULL,
   tstamp timestamp without time zone NOT NULL,
   changeset_id bigint NOT NULL,
@@ -68,7 +66,6 @@ CREATE TABLE ways (
   id bigint NOT NULL,
   version int NOT NULL,
   visible boolean NOT NULL,
-  current boolean NOT NULL,
   user_id int NOT NULL,
   tstamp timestamp without time zone NOT NULL,
   changeset_id bigint NOT NULL,
@@ -82,7 +79,6 @@ CREATE TABLE relations (
   version int NOT NULL,
   rev int NOT NULL,
   visible boolean NOT NULL,
-  current boolean NOT NULL,
   user_id int NOT NULL,
   tstamp timestamp without time zone NOT NULL,
   changeset_id bigint NOT NULL,