Permalink
Browse files

Remove all remaining traces of the unused service cache.

  • Loading branch information...
1 parent 5f0c23a commit 706b27b24c8a73ea36ab159530f277f00abe02b1 @spladug spladug committed Jan 5, 2012
Showing with 1 addition and 250 deletions.
  1. +0 −113 r2/draw_load.py
  2. +0 −4 r2/example.ini
  3. +1 −13 r2/r2/lib/app_globals.py
  4. +0 −8 r2/r2/lib/manager/db_manager.py
  5. +0 −112 r2/supervise_watcher.py
View
@@ -1,113 +0,0 @@
-from __future__ import with_statement
-import Image, ImageDraw
-
-colors = [
- "#FFFFFF", "#f0f5FF",
- "#E2ECFF", "#d6f5cb",
- "#CAFF98", "#e4f484",
- "#FFEA71", "#ffdb81",
- "#FF9191", "#FF0000"]
-
-def get_load_level(host, nlevels = 8):
- # default number of cpus shall be 1
- ncpus = getattr(host, "ncpu", 1)
- # color code in nlevel levels
- return _load_int(host.load(), ncpus, nlevels = 8)
-
-def _load_int(current, max_val, nlevels = 8):
- i = min(max(int(nlevels*current/max_val+0.4), 0),nlevels+1)
- return colors[i]
-
-def draw_load(row_size = 12, width = 200, out_file = "/tmp/load.png"):
- from r2.lib import services
-
- a = services.AppServiceMonitor()
- hosts = list(a)
-
- number = (len([x for x in hosts if x.services]) +
- len([x for x in hosts if x.database]) +
- sum(len(x.queue.queues) for x in hosts if x.queue)) + 14
-
- im = Image.new("RGB", (width, number * row_size + 3))
- draw = ImageDraw.Draw(im)
- def draw_box(label, color, center = False):
- ypos = draw_box.ypos
- xpos = 1
- if center:
- w, h = draw.textsize(label)
- xpos = (width - w) / 2
- draw.rectangle(((1, ypos), (width-2, ypos + row_size)), color)
- draw.text((xpos,ypos+1), label, fill = "#000000")
- draw_box.ypos += row_size
- draw_box.ypos = 0
-
- draw_box(" ==== DATABASES ==== ", "#BBBBBB", center = True)
- for host in hosts:
- if host.database or host.host.startswith('vote'):
- draw_box(" %s load: %s" % (host.host, host.load()),
- get_load_level(host))
-
- draw_box(" ==== SERVICES ==== ", "#BBBBBB", center = True)
- for host in hosts:
- if host.host.startswith('app'):
- draw_box(" %s load: %s" % (host.host, host.load()),
- get_load_level(host))
-
- draw_box(" ==== BUTTONS ==== ", "#BBBBBB", center = True)
- for host in hosts:
- if host.host.startswith('button'):
- draw_box(" %s load: %s" % (host.host, host.load()),
- get_load_level(host))
-
- draw_box(" ==== MEDIA ==== ", "#BBBBBB", center = True)
- for host in hosts:
- if host.host.startswith('media'):
- draw_box(" %s load: %s" % (host.host, host.load()),
- get_load_level(host))
- draw_box(" ==== SEARCH ==== ", "#BBBBBB", center = True)
- for host in hosts:
- if host.host.startswith('search'):
- draw_box(" %s load: %s" % (host.host, host.load()),
- get_load_level(host))
-
- draw_box(" ==== CACHES ==== ", "#BBBBBB", center = True)
- for host in hosts:
- if (host.host.startswith('cache') or host.host.startswith('pmc') or
- host.host.startswith('url')):
- draw_box(" %s load: %s" % (host.host, host.load()),
- get_load_level(host))
-
- draw_box(" ==== QUEUES ==== ", "#BBBBBB", center = True)
- for host in hosts:
- if host.queue:
- draw_box(" %s load: %s" % (host.host, host.load()),
- get_load_level(host))
- for name, data in host.queue:
- max_len = host.queue.max_length(name)
- draw_box(" %16s: %5s / %5s" % (name, data(), max_len),
- _load_int(data(), max_len))
-
- with open(out_file, 'w') as handle:
- im.save(handle, "PNG")
-
-
-def merge_images(out_file, file_names):
- images = []
- width = 0
- height = 0
- for f in file_names:
- images.append(Image.open(f))
- w, h = images[-1].size
- width = max(w, width)
- height += h
-
- total = Image.new("RGB", (width, height))
- height = 0
- for im in images:
- total.paste(im, (0, height))
- w, h = im.size
- height += h
-
- with open(out_file, 'w') as handle:
- total.save(out_file)
-
View
@@ -85,8 +85,6 @@ timezone = UTC
display_timezone = MST
# secret key for accessing /shutdown
shutdown_secret = 12345
-# list of servers that the service monitor will care about
-monitored_servers = reddit, localhost
# https api endpoint (must be g.domain or a subdomain of g.domain)
https_endpoint =
# name of the cookie to drop with login information
@@ -110,8 +108,6 @@ stalecaches =
# render caches (the second is "remote" and the local is optional but in the same format)
local_rendercache =
rendercaches = 127.0.0.1:11211
-# cache for storing service monitor information
-servicecaches = 127.0.0.1:11211
# -- permacache options --
# permacache is memcaches -> cassanda -> memcachedb
View
@@ -104,11 +104,9 @@ class Globals(object):
'memcaches',
'permacache_memcaches',
'rendercaches',
- 'servicecaches',
'cassandra_seeds',
'admins',
'sponsors',
- 'monitored_servers',
'automatic_reddits',
'agents',
'allowed_css_linked_domains',
@@ -278,11 +276,6 @@ def setup(self, global_conf):
num_clients = num_mc_clients)))
self.cache_chains.update(rendercache=self.rendercache)
- self.servicecache = MemcacheChain((localcache_cls(),
- CMemcache(self.servicecaches,
- num_clients = num_mc_clients)))
- self.cache_chains.update(servicecache=self.servicecache)
-
self.thing_cache = CacheChain((localcache_cls(),))
self.cache_chains.update(thing_cache=self.thing_cache)
@@ -400,8 +393,6 @@ def to_iter(v, delim = ','):
return (x.strip() for x in v.split(delim) if x)
def load_db_params(self, gc):
- from r2.lib.services import get_db_load
-
self.databases = tuple(self.to_iter(gc['databases']))
self.db_params = {}
if not self.databases:
@@ -425,10 +416,7 @@ def load_db_params(self, gc):
if params['max_overflow'] == "*":
params['max_overflow'] = self.db_pool_overflow_size
- ip = params['db_host']
- ip_loads = get_db_load(self.servicecache, ip)
- if ip not in ip_loads or ip_loads[ip][0] < 1000:
- dbm.setup_db(db_name, g_override=self, **params)
+ dbm.setup_db(db_name, g_override=self, **params)
self.db_params[db_name] = params
dbm.type_db = dbm.get_engine(gc['type_db'])
@@ -78,13 +78,8 @@ def rels_iter(self):
yield name, (type1_name, type2_name, engines)
def mark_dead(self, engine, g_override=None):
- from r2.lib import services
logger.error("db_manager: marking connection dead: %r" % engine)
self.dead[engine] = time.time()
- if g_override is None:
- services.AppServiceMonitor.mark_db_down(engine.url.host)
- else:
- services.mark_db_down(g_override.servicecache, engine.url.host)
def test_engine(self, engine, g_override=None):
try:
@@ -106,10 +101,7 @@ def get_engines(self, names):
return [self._engines[name] for name in names if name in self._engines]
def get_read_table(self, tables):
- from r2.lib.services import AppServiceMonitor
- # short-cut for only one element
if len(tables) == 1:
return tables[0]
-
return random.choice(list(tables))
View
@@ -1,112 +0,0 @@
-# The contents of this file are subject to the Common Public Attribution
-# License Version 1.0. (the "License"); you may not use this file except in
-# compliance with the License. You may obtain a copy of the License at
-# http://code.reddit.com/LICENSE. The License is based on the Mozilla Public
-# License Version 1.1, but Sections 14 and 15 have been added to cover use of
-# software over a computer network and provide for limited attribution for the
-# Original Developer. In addition, Exhibit A has been modified to be consistent
-# with Exhibit B.
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
-# the specific language governing rights and limitations under the License.
-#
-# The Original Code is Reddit.
-#
-# The Original Developer is the Initial Developer. The Initial Developer of the
-# Original Code is CondeNet, Inc.
-#
-# All portions of the code written by CondeNet are Copyright (c) 2006-2010
-# CondeNet, Inc. All Rights Reserved.
-################################################################################
-import os, re, sys, time, smtplib
-from r2.lib.services import AppServiceMonitor
-from pylons import g
-
-nerds_email = g.nerds_email
-smtp_server = g.smtp_server
-
-def Alert(restart_list = ['MEM','CPU'],
- alert_recipients = [nerds_email],
- alert_sender = nerds_email,
- cpu_limit = 99, mem_limit = 5,
- smtpserver = smtp_server, test = False):
-
- p = re.compile("newreddit(\d+)")
- cache_key = 'already_alerted_'
- for host in AppServiceMonitor(g.monitored_servers):
- for service in host:
- # cpu values
- cpu = [service.cpu(x) for x in (0, 5, 60, 300)]
-
- output = "\nCPU: " + ' '.join("%6.2f%%" % x for x in cpu)
- output += "\nMEMORY: %6.2f%%" % service.mem()
-
- service_name = "%s %s" % (host.host, service.name)
-
- # is this service pegged?
- mem_pegged = ('MEM' in restart_list and service.mem() > mem_limit)
- need_restart = (('CPU' in restart_list and
- all(x >= cpu_limit for x in cpu)) or mem_pegged)
-
-
- if (need_restart):
- mesg = ("To: " + ', '.join(alert_recipients) +
- "\nSubject: " + service_name +" needs attention\n\n"
- + service_name
- + (" is out of mem: " if mem_pegged else " is pegged:" )
- + output)
- m = p.match(service.name)
- # If we can restart this process, we do it here
- if m:
- proc_number = str(m.groups()[0])
- cmd = "/usr/local/bin/push -h " + \
- host.host + " -r " + proc_number
- if test:
- print ("would have restarted the app with command '%s'"
- % cmd)
- else:
- result = os.popen3(cmd)[2].read()
- # We override the other message to show we restarted it
- mesg = ("To: nerds@gmail.com\n" +
- "Subject: " + "Process " +
- proc_number + " on " + host.host +
- " was automatically restarted " +
- "due to the following:\n\n" +
- output + "\n\n" +
- "Here was the output:\n" + result)
- # Uncomment this to disable restart messages
- #mesg = ""
- last_alerted = g.rendercache.get(cache_key + service_name) or 0
- #last_alerted = 0
- if mesg is not "":
- if test:
- print "would have sent email\n '%s'" % mesg
- elif (time.time() - last_alerted > 300):
- g.rendercache.set(cache_key + service_name, time.time())
- session = smtplib.SMTP(smtpserver)
- smtpresult = session.sendmail(alert_sender,
- alert_recipients,
- mesg)
- session.quit()
-
-def Run(srvname, *a, **kw):
- args = {}
- if kw.has_key('queue_length_max'):
- args['queue_length_max'] = kw.pop('queue_length_max')
- AppServiceMonitor(**args).monitor(srvname, *a, **kw)
-
-def Test(num, load = 1., pid = 0):
- services = Services()
- for i in xrange(num):
- name = 'testproc' + str(i)
- p = i or pid
- services.add(name, p, "10")
-
- services.track(p, 100. * (i+1) / (num),
- 20. * (i+1) / num, 1.)
- services.load = load
- services.set_cache()
-
-if __name__ == '__main__':
- Run(sys.argv[1:] if sys.argv[1:] else ['newreddit'])

0 comments on commit 706b27b

Please sign in to comment.