CBG-241 - Remove duplicate sgcollect_info files (#3995)

* CBG-241: Add _status Admin endpoint

* CBG-241: Simplify _config and status in sgcollect

Also moved these from individual files into sync_gateway.log to better
align with cbcollect_info

* CBG-241: Add test for DBRoot

* Add reusable JSON response checking

* Gofmt run

* CBG-241: Fix JSON in sync_gateway.log

* Address PR comments

* RestTester fix

* Addressed comments

* Fixed JSON typos

rest/admin_api.go

@@ -21,7 +21,7 @@ import (
 	"github.com/couchbase/sync_gateway/auth"
 	"github.com/couchbase/sync_gateway/base"
 	"github.com/couchbase/sync_gateway/db"
-	"github.com/couchbaselabs/sg-replicate"
+	sgreplicate "github.com/couchbaselabs/sg-replicate"
 	"github.com/gorilla/mux"
 )
 
@@ -340,6 +340,56 @@ func (h *handler) handleGetLogging() error {
 	return nil
 }
 
+type DatabaseStatus struct {
+	SequenceNumber uint64 `json:"seq"`
+	ServerUUID     string `json:"server_uuid"`
+	State          string `json:"state"`
+}
+
+type Vendor struct {
+	Name    string `json:"name"`
+	Version string `json:"version"`
+}
+
+type Status struct {
+	Databases   map[string]DatabaseStatus `json:"databases"`
+	ActiveTasks []base.Task               `json:"active_tasks"`
+	Version     string                    `json:"version"`
+	Vendor      Vendor                    `json:"vendor"`
+}
+
+func (h *handler) handleGetStatus() error {
+
+	var status = Status{
+		Databases:   make(map[string]DatabaseStatus),
+		ActiveTasks: h.server.replicator.ActiveTasks(),
+		Version:     base.LongVersionString,
+		Vendor: Vendor{
+			Name:    base.ProductName,
+			Version: base.VersionNumber,
+		},
+	}
+
+	for _, database := range h.server.databases_ {
+		lastSeq := uint64(0)
+		runState := db.RunStateString[atomic.LoadUint32(&database.State)]
+
+		// Don't bother trying to lookup LastSequence() if offline
+		if runState != db.RunStateString[db.DBOffline] {
+			lastSeq, _ = database.LastSequence()
+		}
+
+		status.Databases[database.Name] = DatabaseStatus{
+			SequenceNumber: lastSeq,
+			State:          runState,
+			ServerUUID:     database.GetServerUUID(),
+		}
+	}
+
+	h.writeJSON(status)
+	return nil
+}
+
 func (h *handler) handleSetLogging() error {
 	body, err := h.readBody()
 	if err != nil {

rest/admin_api_test.go

@@ -565,6 +565,27 @@ func TestLoggingCombined(t *testing.T) {
 	goassert.DeepEquals(t, logKeys, map[string]bool{"Changes": true, "Cache": true, "HTTP": true})
 }
 
+func TestGetStatus(t *testing.T) {
+	rtConfig := RestTesterConfig{NoFlush: true}
+	rt := NewRestTester(t, &rtConfig)
+	defer rt.Close()
+
+	response := rt.SendRequest("GET", "/_status", "")
+	assertStatus(t, response, 404)
+
+	response = rt.SendAdminRequest("GET", "/_status", "")
+	assertStatus(t, response, 200)
+	var responseBody Status
+	err := json.Unmarshal(response.Body.Bytes(), &responseBody)
+	assert.NoError(t, err)
+
+	goassert.Equals(t, responseBody.Version, base.LongVersionString)
+
+	response = rt.SendAdminRequest("OPTIONS", "/_status", "")
+	assertStatus(t, response, 204)
+	goassert.Equals(t, response.Header().Get("Allow"), "GET")
+}
+
 // Test user delete while that user has an active changes feed (see issue 809)
 func TestUserDeleteDuringChangesWithAccess(t *testing.T) {
 

rest/api.go

@@ -21,7 +21,7 @@ import (
 	"strconv"
 	"sync/atomic"
 
-	"github.com/couchbase/sg-bucket"
+	sgbucket "github.com/couchbase/sg-bucket"
 	"github.com/couchbase/sync_gateway/base"
 	"github.com/couchbase/sync_gateway/db"
 )
@@ -186,6 +186,18 @@ func (h *handler) instanceStartTime() json.Number {
 	return json.Number(strconv.FormatInt(h.db.StartTime.UnixNano()/1000, 10))
 }
 
+type DatabaseRoot struct {
+	DBName                        string      `json:"db_name"`
+	SequenceNumber                uint64      `json:"update_seq"`
+	CommittedUpdateSequenceNumber uint64      `json:"committed_update_seq"`
+	InstanceStartTime             json.Number `json:"instance_start_time"`
+	CompactRunning                bool        `json:"compact_running"`
+	PurgeSequenceNumber           uint64      `json:"purge_seq"`
+	DiskFormatVersion             uint64      `json:"disk_format_version"`
+	State                         string      `json:"state"`
+	ServerUUID                    string      `json:"server_uuid,omitempty"`
+}
+
 func (h *handler) handleGetDB() error {
 	if h.rq.Method == "HEAD" {
 		return nil
@@ -199,19 +211,16 @@ func (h *handler) handleGetDB() error {
 		lastSeq, _ = h.db.LastSequence()
 	}
 
-	response := db.Body{
-		"db_name":              h.db.Name,
-		"update_seq":           lastSeq,
-		"committed_update_seq": lastSeq,
-		"instance_start_time":  h.instanceStartTime(),
-		"compact_running":      h.db.IsCompactRunning(),
-		"purge_seq":            0, // TODO: Should track this value
-		"disk_format_version":  0, // Probably meaningless, but add for compatibility
-		"state":                runState,
-	}
-
-	if uuid := h.db.DatabaseContext.GetServerUUID(); uuid != "" {
-		response["server_uuid"] = uuid
+	var response = DatabaseRoot{
+		DBName:                        h.db.Name,
+		SequenceNumber:                lastSeq,
+		CommittedUpdateSequenceNumber: lastSeq,
+		InstanceStartTime:             h.instanceStartTime(),
+		CompactRunning:                h.db.IsCompactRunning(),
+		PurgeSequenceNumber:           0, // TODO: Should track this value
+		DiskFormatVersion:             0, // Probably meaningless, but add for compatibility
+		State:                         runState,
+		ServerUUID:                    h.db.DatabaseContext.GetServerUUID(),
 	}
 
 	h.writeJSON(response)

rest/api_test.go

@@ -68,6 +68,27 @@ func TestRoot(t *testing.T) {
 	goassert.Equals(t, response.Header().Get("Allow"), "GET, HEAD")
 }
 
+func TestDBRoot(t *testing.T) {
+	rtConfig := RestTesterConfig{NoFlush: true}
+	rt := NewRestTester(t, &rtConfig)
+	defer rt.Close()
+
+	response := rt.SendRequest("GET", "/db/", "")
+	assertStatus(t, response, 200)
+	var body db.Body
+	err := json.Unmarshal(response.Body.Bytes(), &body)
+	assert.NoError(t, err)
+
+	goassert.Equals(t, body["db_name"], "db")
+	goassert.Equals(t, body["state"], "Online")
+
+	response = rt.SendRequest("HEAD", "/db/", "")
+	assertStatus(t, response, 200)
+	response = rt.SendRequest("OPTIONS", "/db/", "")
+	assertStatus(t, response, 204)
+	goassert.Equals(t, response.Header().Get("Allow"), "GET, HEAD, POST, PUT")
+}
+
 func (rt *RestTester) createDoc(t *testing.T, docid string) string {
 	response := rt.SendRequest("PUT", "/db/"+docid, `{"prop":true}`)
 	assertStatus(t, response, 201)

rest/routing.go

@@ -215,6 +215,8 @@ func CreateAdminRouter(sc *ServerContext) *mux.Router {
 		makeOfflineHandler(sc, adminPrivs, (*handler).handleReplicate)).Methods("POST")
 	r.Handle("/_active_tasks",
 		makeOfflineHandler(sc, adminPrivs, (*handler).handleActiveTasks)).Methods("GET")
+	r.Handle("/_status",
+		makeHandler(sc, adminPrivs, (*handler).handleGetStatus)).Methods("GET")
 
 	r.Handle("/_sgcollect_info",
 		makeHandler(sc, adminPrivs, (*handler).handleSGCollectStatus)).Methods("GET")

tools/password_remover.py

@@ -168,7 +168,8 @@ def remove_passwords(json_text, log_json_parsing_exceptions=True):
 
         remove_passwords_from_config(parsed_json)
 
-        formatted_json_string = json.dumps(parsed_json, indent=4)
+        # Append a trailing \n here to ensure there's adequate separation in sync_gateway.log
+        formatted_json_string = json.dumps(parsed_json, indent=4) + "\n"
 
         return formatted_json_string
 
@@ -179,6 +180,18 @@ def remove_passwords(json_text, log_json_parsing_exceptions=True):
         return '{"Error":"Error in sgcollect_info password_remover.py trying to remove passwords.  See logs for details"}'
 
 
+def pretty_print_json(json_text):
+    """
+    Content postprocessor that pretty prints JSON.
+    Returns original string with a trailing \n (to ensure separation in sync_gateway.log) if formatting fails
+    """
+    try:
+        json_text = json.dumps(json.loads(json_text), indent=4)
+    except Exception as e:
+        print("Exception trying to parse JSON {0}.  Exception: {1}".format(json_text, e))
+    return json_text + "\n"
+
+
 def strip_password_from_url(url_string):
     """
     Given a URL string like:

tools/sgcollect_info

@@ -354,36 +354,6 @@ def get_db_list(sg_url):
     # return list of dbs
     return data
 
-
-# Get the "status" for the server overall that's available
-# at the server root URL, as well as the status of each database
-def make_status_tasks(sg_url, should_redact):
-
-    tasks = []
-
-    # Get server config
-    task = make_curl_task(name="Collect server status",
-                          user="",
-                          password="",
-                          url=sg_url,
-                          log_file="server_status.log")
-    tasks.append(task)
-
-    # Get list of dbs from _all_dbs
-    # For each db, get db config
-    dbs = get_db_list(sg_url)
-    for db in dbs:
-        db_status_url = "{0}/{1}".format(sg_url, db)
-        task = make_curl_task(name="Collect db status for db: {0}".format(db),
-                              user="",
-                              password="",
-                              url=db_status_url,
-                              log_file="db_{0}_status.log".format(db))
-        tasks.append(task)
-
-    return tasks
-
-
 # Startup config
 #   Commandline args (covered in expvars, IIRC)
 #   json file.
@@ -429,27 +399,14 @@ def make_config_tasks(zip_dir, sg_config_path, sg_url, should_redact):
 
     # Get server config
     server_config_url = "{0}/_config".format(sg_url)
-    config_task = make_curl_task(name="Collect server config",
+    config_task = make_curl_task(name="Running server config",
                                  user="",
                                  password="",
                                  url=server_config_url,
-                                 log_file="running_server_config.log",
+                                 log_file="sync_gateway.log",
                                  content_postprocessors=server_config_postprocessors)
     collect_config_tasks.append(config_task)
 
-    # Get list of dbs from _all_dbs
-    # For each db, get db config
-    dbs = get_db_list(sg_url)
-    for db in dbs:
-        db_config_url = "{0}/{1}/_config".format(sg_url, db)
-        config_task = make_curl_task(name="Collect db config for db: {0}".format(db),
-                                     user="",
-                                     password="",
-                                     url=db_config_url,
-                                     log_file="running_db_{0}_config.log".format(db),
-                                     content_postprocessors=db_config_postprocessors)
-        collect_config_tasks.append(config_task)
-
     return collect_config_tasks
 
 
@@ -543,8 +500,13 @@ def make_sg_tasks(zip_dir, sg_url, sync_gateway_config_path_option, sync_gateway
     # Add a task to collect Sync Gateway config
     config_tasks = make_config_tasks(zip_dir, sg_config_path, sg_url, should_redact)
 
-    # Curl the / endpoint and /db endpoints and save output
-    status_tasks = make_status_tasks(sg_url, should_redact)
+    # Curl the /_status
+    status_tasks = make_curl_task(name="Collect server status",
+                                  user="",
+                                  password="",
+                                  url="{0}/_status".format(sg_url),
+                                  log_file="sync_gateway.log",
+                                  content_postprocessors=[password_remover.pretty_print_json])
 
     # Compbine all tasks into flattened list
     sg_tasks = flatten(