Merge pull request #25 from dimagi/dmr/include-private-dbs-in-shard-m…

…igrations

Include private dbs in shard migrations

.travis.yml

@@ -5,6 +5,7 @@ python: "2.7"
 before_install:
   - git clone https://github.com/sstephenson/bats.git
   - cd bats
+  - git checkout v0.4.0
   - sudo ./install.sh /usr/local
   - cd -
 install:

couchdb_cluster_admin/add_replica_node.py

@@ -71,7 +71,7 @@ def _add_node(node_details, new_node):
         print(line)
         sys.exit(1)
 
-    for db_name in get_db_list(node_details, skip_private=False):
+    for db_name in get_db_list(node_details):
         if db_name.startswith('_'):
             # TODO: remove this once there's a workaround for https://github.com/apache/couchdb/issues/858
             print('Skipping {}'.format(db_name))

couchdb_cluster_admin/copy_db_to_new_cluster.py

@@ -95,7 +95,7 @@ def _copy_db_doc(from_details, to_details, db_name):
         sys.exit(1)
 
     if args.database == 'ALL':
-        dbs = get_db_list(from_details, skip_private=False)
+        dbs = get_db_list(from_details)
     else:
         dbs = [args.database]
     print(dbs)

couchdb_cluster_admin/file_plan.py

@@ -58,8 +58,12 @@ def figure_out_what_you_can_and_cannot_delete(plan, shard_suffix_by_db_name=None
                 all_filenames.add(couch_file_name)
 
                 view_file = Nodefile(db_name, node, shard, view_file_name)
-                important_files_by_node[node].add(view_file)
-                all_filenames.add(view_file_name)
+                # _global_changes doesn't have any views, so there's no view file
+                # The same is true of _any_ db with no views, but it's rare enough
+                # that I'm just cutting corners for simplicity
+                if db_name != '_global_changes':
+                    important_files_by_node[node].add(view_file)
+                    all_filenames.add(view_file_name)
 
     deletable_files_by_node = {}
     for node, important_files in important_files_by_node.items():

couchdb_cluster_admin/remove_node.py

@@ -71,7 +71,7 @@ def _remove_node(node_details, new_node):
         sys.exit(0)
 
     remove_from_cluster = True
-    for db_name in get_db_list(node_details, skip_private=False):
+    for db_name in get_db_list(node_details):
         if db_name.startswith('_'):
             # TODO: remove this once there's a workaround for https://github.com/apache/couchdb/issues/858
             print("Skipping db {}".format(db_name))

couchdb_cluster_admin/suggest_shard_allocation.py

@@ -2,6 +2,7 @@
 from collections import defaultdict
 import json
 
+import requests
 from memoized import memoized_property
 
 from utils import humansize, get_arg_parser, get_config_from_args, check_connection, \
@@ -377,7 +378,14 @@ def main():
 
     if args.commit:
         for shard_allocation_doc in shard_allocations:
-            print put_shard_allocation(config, shard_allocation_doc)
+            db_name = shard_allocation_doc.db_name
+            try:
+                print put_shard_allocation(config, shard_allocation_doc)
+            except requests.exceptions.HTTPError as e:
+                if db_name.startswith('_') and e.response.json().get('error') == 'illegal_docid':
+                    print("Skipping {} (error response was {})".format(db_name, e.response.json()))
+                else:
+                    raise
 
 
 def get_shard_allocation_from_plan(config, plan, create=False):

couchdb_cluster_admin/utils.py

@@ -41,16 +41,9 @@ def _do_request(node_details, path, port, method='get', params=None, json=None):
     return response.json()
 
 
-def get_db_list(node_details, skip_private=True):
-    """
-    :param skip_private: if True, exclude dbs that start with an underscore from the result
-    :return: list of dbs
-    """
+def get_db_list(node_details):
     db_names = do_couch_request(node_details, '_all_dbs')
-    if skip_private:
-        return [db_name for db_name in db_names if not db_name.startswith('_')]
-    else:
-        return db_names
+    return db_names
 
 
 def get_db_metadata(node_details, db_name):

test/test.bats

@@ -4,6 +4,12 @@ function setup {
     db_name=test$(head -n20 /dev/random | python -c 'import hashlib, sys; print hashlib.md5(sys.stdin.read()).hexdigest()')
     echo "PUT" $db_name
     curl -sX PUT http://localhost:15984/${db_name}
+    curl -sX PUT http://localhost:15984/_users
+    curl -sX PUT http://localhost:15984/_users/org.couchdb.user:jan \
+     -H "Accept: application/json" \
+     -H "Content-Type: application/json" \
+     -d '{"name": "jan", "password": "apple", "roles": [], "type": "user"}'
+
 }
 
 function doccount {
@@ -65,6 +71,16 @@ function wait_for_couch_ping {
 
     python couchdb_cluster_admin/suggest_shard_allocation.py --conf=test/local.yml --from-plan=test/local.plan.json --commit-to-couchdb
 
+    echo $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount)
+    sleep 5
+    echo $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount)
+    sleep 5
+    echo $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount)
+    sleep 5
+    echo $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount)
+    sleep 5
+    echo $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount)
+    sleep 5
     echo $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount) $(doccount)
     [ "$(doccount)" = '10' ]
     echo $(longform_doccount) $(longform_doccount) $(longform_doccount)