MANTA-3284 want operator-configurable throttle on muskie requests

README.md

@@ -5,7 +5,7 @@
 -->
 
 <!--
-    Copyright (c) 2017, Joyent, Inc.
+    Copyright (c) 2018, Joyent, Inc.
 -->
 
 # manta-muskie: The Manta WebAPI
@@ -226,3 +226,55 @@ enumerate the Muskie instances using DNS, nor is there a way to add that without
 changing the DNS name for webapi instances, which would be a flag day for
 Muppet.**  (This may explain why [muppet](https://github.com/joyent/muppet) is a
 ZooKeeper consumer rather than just a DNS client.)
+
+## Dtrace Probes
+
+Muskie has two dtrace providers. The first, `muskie`, has the following probes:
+* `client_close`: `json`. Fires if a client uploading an object or part closes
+  before data has been streamed to mako. Also fires if the client closes the
+  connection while the stream is in progress. The argument json object has the
+  following format:
+  ```
+  {
+      id: restify uuid, or x-request-id/request-id http header (string)
+      method: request http method (string)
+      headers: http headers specified by the client (object)
+      url: http request url (string)
+      bytes_sent: number of bytes streamed to mako before client close (int)
+      bytes_expected: number of bytes that should have been streamed (int)
+  }
+  ```
+* `socket_timeout`: `json`. Fires when the timeout limit is reached on a
+  connection to a client. This timeout can be configured either by setting the
+  `SOCKET_TIMEOUT` environment variable. The default is 120 seconds. The object
+  passed has the same fields to the `client_close` dtrace probe, except for the
+  `bytes_sent` and `bytes_expected`. These parameters are only present if muskie
+  is able to determine the last request sent on this socket.
+
+The second provider, `muskie-throttle`, has the following probes, which will not
+fire if the throttle is disabled:
+* `request_throttled`: `int`, `int`, `char *`, `char *` - slots occupied, queued
+  requests, url, method. Fires when a request has been throttled.
+* `request_handled`: `int`, `int`, `char *`, `char *` - slots occupied, queued
+  requests, url, method. Fires after a request has been handled.
+Internally, the muskie throttle is implemented with a vasync-queue. A "slot"
+in the above description refers to one of `concurrency` possible spaces
+allotted for concurrently scheduled request-handling callbacks. If all slots are
+occupied, incoming requests will be "queued", which indicates that they are
+waiting for slots to free up.
+* `queue_enter`: `char *` - restify request uuid. This probe fires as a request
+enters the queue.
+* `queue_leave`: `char *` - restify request uuid. This probe fires as a request
+is dequeued, before it is handled. The purpose of these probes is to make it
+easy to write d scripts that measure the latency impact the throttle has on
+individual requests.
+
+The script `bin/throttlestat.d` is implemented as an analog to `moraystat.d`
+with the `queue_enter` and `queue_leave` probes. It is a good starting point for
+gaining insight into both how actively a muskie process is being throttled and
+how much stress it is under.
+
+The throttle probes are provided in a separate provider to prevent coupling the
+throttle implementation with muskie itself. Future work may involve making the
+throttle a generic module that can be included in any service with minimal code
+modification.

bin/throttlestat.d

@@ -0,0 +1,60 @@
+#!/usr/sbin/dtrace -Cs
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+/*
+ * Copyright (c) 2018, Joyent, Inc.
+ */
+
+#pragma D option quiet
+
+int latencies[char *];
+
+muskie-throttle*:::queue_enter
+{
+	latencies[copyinstr(arg0)] = timestamp;
+}
+
+muskie-throttle*:::queue_leave
+/latencies[copyinstr(arg0)]/
+{
+	latencies[copyinstr(arg0)] = timestamp - latencies[copyinstr(arg0)];
+	@avg_latency[pid] = avg(latencies[copyinstr(arg0)] / 1000000);
+	latencies[copyinstr(arg0)] = 0;
+}
+
+muskie-throttle*:::request_throttled
+{
+	@throttled[pid] = count();
+}
+
+muskie-throttle*:::request_handled
+{
+	@qlen[pid] = max(arg1);
+	@qrunning[pid] = max(arg0);
+}
+
+profile:::tick-1sec
+/lines < 1/
+{
+	printf("THROTTLED-PER-SEC | AVG-LATENCY-MS | MAX-QLEN | MAX-RUNNING\n");
+	printf("------------------+----------------+----------+------------\n");
+	lines = 5;
+}
+
+
+profile:::tick-1sec
+/lines > 0/
+{
+	lines -= 1;
+	printa("%@4u              %@4u            %@4u       %@4u\n", @throttled,
+            @avg_latency, @qlen, @qrunning);
+
+	clear(@throttled);
+	clear(@avg_latency);
+	clear(@qlen);
+	clear(@qrunning);
+}

etc/config.coal.json

@@ -7,6 +7,11 @@
             "type": "udp"
         }
     },
+    "throttle": {
+        "enabled": false,
+        "concurrency": 50,
+        "queueTolerance": 25
+    },
     "maxObjectCopies": 6,
     "maxRequestAge": 600,
     "enableMPU": true,

lib/errors.js

@@ -5,7 +5,7 @@
  */
 
 /*
- * Copyright (c) 2017, Joyent, Inc.
+ * Copyright (c) 2018, Joyent, Inc.
  */
 
 var assert = require('assert');
@@ -594,6 +594,15 @@ function SSLRequiredError() {
 }
 util.inherits(SSLRequiredError, MuskieError);
 
+function ThrottledError() {
+    MuskieError.call(this, {
+        restCode: 'ThrottledError',
+        statusCode: 503,
+        message: 'manta throttled this request'
+    });
+    this.name = this.constructor.name;
+}
+util.inherits(ThrottledError, MuskieError);
 
 function UploadAbandonedError() {
     MuskieError.call(this, {
@@ -654,6 +663,12 @@ function translateError(err, req) {
             'name': cause.context.name
         }, '%s', cause.context.message)));
     }
+
+    cause = VError.findCauseByName(err, 'ThrottledError');
+    if (cause !== null) {
+        return (cause);
+    }
+
     cause = VError.findCauseByName(err, 'ObjectNotFoundError');
     if (cause !== null) {
         return (new restify.ResourceNotFoundError(cause,

lib/server.js

@@ -5,7 +5,7 @@
  */
 
 /*
- * Copyright (c) 2017, Joyent, Inc.
+ * Copyright (c) 2018, Joyent, Inc.
  */
 
 var crypto = require('crypto');
@@ -30,6 +30,7 @@ var obj = require('./obj');
 var other = require('./other');
 var picker = require('./picker');
 var uploads = require('./uploads');
+var throttle = require('./throttle');
 
 // injects into the global namespace
 require('./errors');
@@ -100,6 +101,7 @@ function createServer(options, clients, name) {
     assert.object(options, 'options');
     assert.object(options.log, 'options.log');
     assert.object(options.collector, 'options.collector');
+    assert.object(options.throttle, 'options.throttle');
     assert.object(clients, 'clients');
     assert.string(name, 'name');
 
@@ -215,18 +217,23 @@ function createServer(options, clients, name) {
         var errors = [];
 
         if (!clients.picker) {
+            errors.push(new Error('picker unavailable'));
             req.log.error('picker unavailable');
             ok = false;
         } else if (!clients.moray) {
+            errors.push(new Error('index moray unavailable'));
             req.log.error('index moray unavailable');
             ok = false;
         } else if (!clients.mahi) {
+            errors.push(new Error('mahi unavailable'));
             req.log.error('mahi unavailable');
             ok = false;
         } else if (!clients.marlin) {
+            errors.push(new Error('marlin available'));
             req.log.error('marlin unavailable');
             ok = !req.isMarlinRequest();
         } else if (!clients.medusa) {
+            errors.push(new Error('medusa unavailable'));
             req.log.error('medusa unavailable');
             ok = !req.isMedusaRequest();
         }
@@ -239,6 +246,11 @@ function createServer(options, clients, name) {
         }
     });
 
+    if (options.throttle.enabled) {
+        options.throttle.log = options.log;
+        var throttleHandle = throttle.createThrottle(options.throttle);
+        server.use(throttle.throttleHandler(throttleHandle));
+    }
     server.use(auth.authenticationHandler({
         log: log,
         mahi: clients.mahi,