Install ZooKeeper in dev and make liveconfig/secrets easier

This installs a ZooKeeper server in development and adds two
configuration options. Live config and Secrets can now be sourced from
either ZooKeeper or the local config. This allows local installs to
continue with the easy workflow of just modifying the INI file to test
out changes, while allowing us to develop ZK features locally as well
(such as throttles).

.drone.yml

@@ -43,6 +43,9 @@ compose:
     # NOTE: Using 1.4.21 instead of 1.4.17 due to tag availability.
     image: memcached:1.4.21
 
+  zookeeper:
+    image: jplock/zookeeper:3.4.6
+
 # Build steps are where the setup, compilation, and tests happen.
 build:
   # This is a fat Docker image with the apt dependencies pre-installed.

install/install_services.sh

@@ -40,6 +40,7 @@ haproxy
 nginx
 gunicorn
 redis-server
+zookeeperd
 PACKAGES
 
 ###############################################################################

r2/example.ini

@@ -590,9 +590,15 @@ amqp_virtual_host = /
 
 ############################################ ZOOKEEPER
 # zookeeper is optional at the moment
-zookeeper_connection_string =
-zookeeper_username =
-zookeeper_password =
+zookeeper_connection_string = localhost:2181
+zookeeper_username = reddit
+zookeeper_password = reddit
+
+# these should be one of "config" or "zookeeper"
+# - config: read from the local ini file
+# - zookeeper: read from the zookeeper cluster
+liveconfig_source = config
+secrets_source = config
 
 
 ############################################ EMAIL

r2/r2/lib/app_globals.py

@@ -82,6 +82,7 @@
 )
 from r2.lib.translation import get_active_langs, I18N_PATH
 from r2.lib.utils import config_gold_price, thread_dump
+from r2.lib.zookeeper import connect_to_zookeeper, LiveConfig, LiveList
 
 
 LIVE_CONFIG_NODE = "/config/live"
@@ -325,6 +326,11 @@ class Globals(object):
              'cassandra_wcl',
         ],
 
+        ConfigValue.choice(zookeeper="zookeeper", config="config"): [
+            "liveconfig_source",
+            "secrets_source",
+        ],
+
         ConfigValue.timeinterval: [
             'ARCHIVE_AGE',
             "vote_queue_grace_period",
@@ -679,33 +685,29 @@ def setup(self):
         self.startup_timer.intermediate("configuration")
 
         ################# ZOOKEEPER
-        # for now, zookeeper will be an optional part of the stack.
-        # if it's not configured, we will grab the expected config from the
-        # [live_config] section of the ini file
-        zk_hosts = self.config.get("zookeeper_connection_string")
-        if zk_hosts:
-            from r2.lib.zookeeper import (connect_to_zookeeper,
-                                          LiveConfig, LiveList)
-            zk_username = self.config["zookeeper_username"]
-            zk_password = self.config["zookeeper_password"]
-            self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username,
-                                                             zk_password))
-            self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE)
-            self.secrets = fetch_secrets(self.zookeeper)
-            self.throttles = LiveList(self.zookeeper, "/throttles",
-                                      map_fn=ipaddress.ip_network,
-                                      reduce_fn=ipaddress.collapse_addresses)
+        zk_hosts = self.config["zookeeper_connection_string"]
+        zk_username = self.config["zookeeper_username"]
+        zk_password = self.config["zookeeper_password"]
+        self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username,
+                                                         zk_password))
+
+        self.throttles = LiveList(self.zookeeper, "/throttles",
+                                  map_fn=ipaddress.ip_network,
+                                  reduce_fn=ipaddress.collapse_addresses)
 
-            # close our zk connection when the app shuts down
-            SHUTDOWN_CALLBACKS.append(self.zookeeper.stop)
+        parser = ConfigParser.RawConfigParser()
+        parser.optionxform = str
+        parser.read([self.config["__file__"]])
+
+        if self.config["liveconfig_source"] == "zookeeper":
+            self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE)
         else:
-            self.zookeeper = None
-            parser = ConfigParser.RawConfigParser()
-            parser.optionxform = str
-            parser.read([self.config["__file__"]])
             self.live_config = extract_live_config(parser, self.plugins)
+
+        if self.config["secrets_source"] == "zookeeper":
+            self.secrets = fetch_secrets(self.zookeeper)
+        else:
             self.secrets = extract_secrets(parser)
-            self.throttles = tuple()  # immutable since it's not real
 
         ################# PRIVILEGED USERS
         self.admins = PermissionFilteredEmployeeList(