Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
tree: 9f2a9a7f7a
Fetching contributors…

Octocat-spinner-32-eaf2f5

Cannot retrieve contributors at this time

executable file 148 lines (120 sloc) 4.528 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
#!/usr/bin/env ruby
# Import the stuff that osmosis can not import yet, like user statuses.

require 'pg'
require 'getoptlong'
require 'mechanize'

#default values
USERS_AGREED = "http://planet.openstreetmap.org/users_agreed/users_agreed.txt"
CHANGESETS_AGREED = "http://planet.openstreetmap.org/users_agreed/anon_changesets_agreed.txt"
USER_LIMIT = 286582

def get_url_lines(agent, verbose, url)
  if url.start_with? "http://" then
    puts "Downloading #{url}..." if verbose
    agent.get(url).content
  else
    File.open(url, "r")
  end.lines.
    select {|l| not l.match(/^ *(#|$)/) }.
    map {|l| l.to_i }
end

def usage
  puts <<-EOF
usage: dbimport.rb [OPTIONS...]

Imports important parts of the database for the redaction bot.
This is the bits that osmosis does not import like agreed users
and changesets.

-h, --help:
Show help

--host domain:
Database hostname (localhost)

--database name
Database to import into. (osm)
--user name
Username of the database. (openstreetmap)
--password pass
Password for that user. (openstreetmap)

-v --verbose:
Output information about the actions being taken.

-n --no-action
Send all the commands to the database, but do not commit them.

-u --users-agreed file:
A URL or file with a list of user IDs of those who have agreed.
If this is not specified then it will download the list from osm.org.

-c --changesets-agreed file:
A URL or file with a list of changesets which have been agreed.
This is used only where the owner of the changeset is anonymous.
If this is not specified then it will download the list from osm.org.

-l --user-agreed-limit int:
The user ID below which users may not have agreed. For example,
on the main API this number is 286582 and user IDs >= this are
EOF
end

opts = GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT ],
                      ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
                      ['--no-action', '-n', GetoptLong::NO_ARGUMENT],
                      ['--users-agreed', '-u', GetoptLong::REQUIRED_ARGUMENT],
                      ['--changesets-agreed', '-c', GetoptLong::REQUIRED_ARGUMENT],
                      ['--user-agreed-limit', '-l', GetoptLong::REQUIRED_ARGUMENT],
                      ['--host', GetoptLong::REQUIRED_ARGUMENT],
                      ['--database', GetoptLong::REQUIRED_ARGUMENT],
                      ['--user', GetoptLong::REQUIRED_ARGUMENT],
                      ['--password', GetoptLong::REQUIRED_ARGUMENT])

verbose = false
no_action = false
users_agreed = USERS_AGREED
changesets_agreed = CHANGESETS_AGREED
user_limit = USER_LIMIT
dbhost = 'localhost'
dbname = 'osm'
dbuser = 'openstreetmap'
dbpass = 'openstreetmap'

opts.each do |opt, arg|
  case opt
  when '--help'
    usage
    exit 0
  when '--verbose'
    verbose = true
  when '--no-action'
    no_action = true
  when '--users-agreed'
    users_agreed = arg
  when '--changesets-agreed'
    changesets_agreed = arg
  when '--user-agreed-limit'
    user_limit = arg.to_i
  when '--host'
    dbhost = arg
  when '--database'
    dbname = arg
  when '--user'
    dbuser = arg
  when '--password'
    dbpass = arg
  end
end

if not ARGV.empty?
  puts "Unexpected argument #{ARGV[0]}"
  usage
  exit 0
end

agent = Mechanize.new

PGconn.open(:host => dbhost, :dbname => dbname, :user => dbuser, :password => dbpass).transaction do |dbconn|
  # Set all agreed users to agreed
  get_url_lines(agent, verbose, users_agreed).each_slice(100) do |user_ids|
    sql = "UPDATE users SET terms_seen = true, terms_agreed = \'2012-04-10 00:00:00\' WHERE " \
      + user_ids.map {|id| "id = #{id} or "}.join + "false;"
    dbconn.exec(sql)
  end
  
  # Set all users with an id over the limit to agreed
  dbconn.exec("UPDATE users SET terms_seen = true, terms_agreed = \'2012-04-10 00:00:00\' WHERE id >= #{user_limit};")

  # Insert an anonymous agreeing user as a dummy
  dbconn.exec("INSERT INTO users (email, id, pass_crypt, creation_time, display_name, data_public, terms_seen, terms_agreed) VALUES ('anon_user@example.com', -2, 00000000000000000000000000000000, now(), 'Anonymous agreeing user', false, true, '2012-04-10 00:00:00');")

  # Set all agreed anonymous changesets to agreed user
  get_url_lines(agent, verbose, changesets_agreed).each_slice(100) do |changeset_ids|
    sql = "UPDATE changesets SET user_id = -2 WHERE " \
      + changeset_ids.map {|id| "id = #{id} or "}.join + "false;"
    dbconn.exec(sql)
  end

  raise "No actions commited" if no_action
end


Something went wrong with that request. Please try again.