add a migrate.conf that just has fewer db connections.

twitter-archive · Nov 12, 2010 · 430f66b · 430f66b
1 parent 79b696e
commit 430f66b
Showing 1 changed file with 199 additions and 0 deletions.
diff --git a/config/migrate.conf b/config/migrate.conf
@@ -0,0 +1,199 @@
+admin_port = 9991
+admin_http_port = 9990
+w3c_header_period = 60000
+
+db {
+  username = "$(DB_USERNAME)"
+  password = "$(DB_PASSWORD)"
+  connection_pool {
+    max_wait = 100
+    min_evictable_idle_msec = -1
+    size_max = 2
+    size_min = 2
+    test_idle_msec = 1000
+    test_on_borrow = false
+
+    timeout {
+      initialize = 1000
+      open = 50
+      pool_size = 10
+      queue_size = 10000
+    }
+  }
+
+  disable {
+    error_count = 100
+    seconds = 60
+  }
+
+  timeouts {
+    select = 3000
+    select_modify = 3000
+    execute = 3000
+  }
+}
+
+# use a connection pool of 1 when materializing tables.
+# (the memoizing connection pools will create a new connection pool because there's no db name.)
+materializing_db (inherit="db") {
+  connection_pool (inherit="db.connection_pool") {
+    size_max = 1
+    size_min = 1
+  }
+}
+
+errors {
+  deadlock_retries = 3
+}
+
+log {
+  filename = "/var/log/flock/production.log"
+  level = "info"
+  roll = "hourly"
+  throttle_period_msec = 60000
+  throttle_rate = 10
+  truncate_stack_traces = 100
+
+  w3c {
+    node = "w3c"
+    use_parents = false
+    filename = "/var/log/flock/w3c.log"
+    level = "info"
+    roll = "hourly"
+  }
+
+  stats {
+    node = "stats"
+    use_parents = false
+    level = "info"
+    scribe_category = "flock-stats"
+    scribe_server = "localhost"
+    scribe_max_packet_size = 100
+  }
+
+  bad_jobs {
+    node = "bad_jobs"
+    use_parents = false
+    filename = "/var/log/flock/bad_jobs.log"
+    level = "info"
+    roll = "never"
+  }
+}
+
+gizzard_services {
+  name = "edges"
+  min_threads = 2
+  shard_server_port = 7917
+  job_server_port = 7919
+  client_timeout_msec = 100
+  idle_timeout_sec = 60
+}
+
+edges {
+  server_port = 7915
+  intersection_timeout_ms = 100
+  average_intersection_proportion = 0.1
+  intersection_page_size_max = 4000
+  aggregate_jobs_page_size = 500
+  db_name = "edges"
+
+  w3c = [
+    "action-timing",
+    "db-timing",
+    "connection-pool-release-timing",
+    "connection-pool-reserve-timing",
+    "database-open-timing",
+    "database-close-timing",
+    "db-count-query-default",
+    "x-db-timing-query-default",
+    "kestrel-put-timing",
+    "db-count-select",
+    "db-count-execute",
+    "db-timing-select",
+    "db-timing-execute",
+    "job-success-count",
+    "operation",
+    "arguments"
+  ]
+
+  nameservers (inherit="db") {
+    mapping = "byte_swapper"
+
+    connection_pool {
+      max_wait = 100
+      min_evictable_idle_msec = -1
+      size_max = 1
+      size_min = 1
+      test_idle_msec = 1000
+      test_on_borrow = false
+
+      timeout {
+        initialize = 1000
+        open = 500
+        pool_size = 10
+        queue_size = 10000
+      }
+    }
+    query_timeout_default = 15000
+
+    replicas {
+      ns1 (inherit="db") {
+        hostname = "flockdb001.twitter.com"
+        database = "flock_edges_production"
+      }
+
+      ns2 (inherit="db") {
+        hostname = "flockdb002.twitter.com"
+        database = "flock_edges_production"
+      }
+    }
+  }
+
+  future {
+    pool_size = 100
+    max_pool_size = 100
+    keep_alive_time_seconds = 5
+    timeout_seconds = 6
+  }
+
+  replication {
+    future {
+      pool_size = 100
+      max_pool_size = 100
+      keep_alive_time_seconds = 5
+      timeout_seconds = 6
+    }
+  }
+
+  queue {
+    path = "/var/spool/kestrel"
+    primary {
+      job_queue = "edges_jobs"
+      error_queue = "edges_errors"
+      threads = 32
+      error_limit = 100
+      replay_interval = 900
+      per_flush_item_limit = 1000
+      jitter_rate = 0
+    }
+    copy {
+      job_queue = "copy_jobs"
+      error_queue = "copy_errors"
+      threads = 12
+      error_limit = 1000
+      replay_interval = 60
+      per_flush_item_limit = 1000
+      jitter_rate = 0
+    }
+    slow {
+      job_queue = "edges_slow_jobs"
+      error_queue = "edges_slow_errors"
+      threads = 2
+      error_limit = 100
+      replay_interval = 900
+      per_flush_item_limit = 1000
+      jitter_rate = 0
+    }
+    max_memory_size = 36000000
+  }
+}