This repository has been archived by the owner. It is now read-only.
Permalink
Browse files

Install ZooKeeper in dev and make liveconfig/secrets easier

This installs a ZooKeeper server in development and adds two
configuration options. Live config and Secrets can now be sourced from
either ZooKeeper or the local config. This allows local installs to
continue with the easy workflow of just modifying the INI file to test
out changes, while allowing us to develop ZK features locally as well
(such as throttles).
  • Loading branch information...
spladug committed Jun 14, 2016
1 parent 4107dee commit 0d74afb00cb49886261b3394724bc556e08a1532
Showing with 38 additions and 26 deletions.
  1. +3 −0 .drone.yml
  2. +1 −0 install/install_services.sh
  3. +9 −3 r2/example.ini
  4. +25 −23 r2/r2/lib/app_globals.py
View
@@ -43,6 +43,9 @@ compose:
# NOTE: Using 1.4.21 instead of 1.4.17 due to tag availability.
image: memcached:1.4.21
zookeeper:
image: jplock/zookeeper:3.4.6
# Build steps are where the setup, compilation, and tests happen.
build:
# This is a fat Docker image with the apt dependencies pre-installed.
@@ -40,6 +40,7 @@ haproxy
nginx
gunicorn
redis-server
zookeeperd
PACKAGES
###############################################################################
View
@@ -590,9 +590,15 @@ amqp_virtual_host = /
############################################ ZOOKEEPER
# zookeeper is optional at the moment
zookeeper_connection_string =
zookeeper_username =
zookeeper_password =
zookeeper_connection_string = localhost:2181
zookeeper_username = reddit
zookeeper_password = reddit
# these should be one of "config" or "zookeeper"
# - config: read from the local ini file
# - zookeeper: read from the zookeeper cluster
liveconfig_source = config
secrets_source = config
############################################ EMAIL
View
@@ -82,6 +82,7 @@
)
from r2.lib.translation import get_active_langs, I18N_PATH
from r2.lib.utils import config_gold_price, thread_dump
from r2.lib.zookeeper import connect_to_zookeeper, LiveConfig, LiveList
LIVE_CONFIG_NODE = "/config/live"
@@ -325,6 +326,11 @@ class Globals(object):
'cassandra_wcl',
],
ConfigValue.choice(zookeeper="zookeeper", config="config"): [
"liveconfig_source",
"secrets_source",
],
ConfigValue.timeinterval: [
'ARCHIVE_AGE',
"vote_queue_grace_period",
@@ -679,33 +685,29 @@ def setup(self):
self.startup_timer.intermediate("configuration")
################# ZOOKEEPER
# for now, zookeeper will be an optional part of the stack.
# if it's not configured, we will grab the expected config from the
# [live_config] section of the ini file
zk_hosts = self.config.get("zookeeper_connection_string")
if zk_hosts:
from r2.lib.zookeeper import (connect_to_zookeeper,
LiveConfig, LiveList)
zk_username = self.config["zookeeper_username"]
zk_password = self.config["zookeeper_password"]
self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username,
zk_password))
self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE)
self.secrets = fetch_secrets(self.zookeeper)
self.throttles = LiveList(self.zookeeper, "/throttles",
map_fn=ipaddress.ip_network,
reduce_fn=ipaddress.collapse_addresses)
zk_hosts = self.config["zookeeper_connection_string"]
zk_username = self.config["zookeeper_username"]
zk_password = self.config["zookeeper_password"]
self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username,
zk_password))
self.throttles = LiveList(self.zookeeper, "/throttles",
map_fn=ipaddress.ip_network,
reduce_fn=ipaddress.collapse_addresses)
# close our zk connection when the app shuts down
SHUTDOWN_CALLBACKS.append(self.zookeeper.stop)
parser = ConfigParser.RawConfigParser()
parser.optionxform = str
parser.read([self.config["__file__"]])
if self.config["liveconfig_source"] == "zookeeper":
self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE)
else:
self.zookeeper = None
parser = ConfigParser.RawConfigParser()
parser.optionxform = str
parser.read([self.config["__file__"]])
self.live_config = extract_live_config(parser, self.plugins)
if self.config["secrets_source"] == "zookeeper":
self.secrets = fetch_secrets(self.zookeeper)
else:
self.secrets = extract_secrets(parser)
self.throttles = tuple() # immutable since it's not real
################# PRIVILEGED USERS
self.admins = PermissionFilteredEmployeeList(

0 comments on commit 0d74afb

Please sign in to comment.