Add allocation counter tests handler add/remove (#1785)

Motivation:

Allocation counter tests are good, and we aren't measuring this today.

Modifications:

- Wrote some add/remove tests that use different remove functions.

Result:

Better insight into performance.

IntegrationTests/tests_04_performance/test_01_resources/test_1000_addRemoveHandlers.swift

@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftNIO open source project
+//
+// Copyright (c) 2021 Apple Inc. and the SwiftNIO project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import NIO
+
+fileprivate final class RemovableHandler: ChannelInboundHandler, RemovableChannelHandler {
+    typealias InboundIn = NIOAny
+
+    static let name: String = "RemovableHandler"
+
+    var context: ChannelHandlerContext?
+
+    func handlerAdded(context: ChannelHandlerContext) {
+        self.context = context
+    }
+
+    func handlerRemoved(context: ChannelHandlerContext) {
+        self.context = nil
+    }
+}
+
+@inline(__always)
+private func addRemoveBench(iterations: Int, _ removalOperation: (Channel, RemovableHandler) -> EventLoopFuture<Void>) -> Int {
+    let channel = EmbeddedChannel()
+    defer {
+        _ = try! channel.finish()
+    }
+
+    for _ in 0..<iterations {
+        let handler = RemovableHandler()
+        try! channel.pipeline.syncOperations.addHandler(handler, name: RemovableHandler.name)
+        try! removalOperation(channel, handler).wait()
+    }
+
+    return iterations
+}
+
+func run(identifier: String) {
+    measure(identifier: identifier + "_handlertype") {
+        addRemoveBench(iterations: 1000) { channel, handler in
+            channel.pipeline.removeHandler(handler)
+        }
+    }
+
+    measure(identifier: identifier + "_handlername") {
+        addRemoveBench(iterations: 1000) { channel, _ in
+            channel.pipeline.removeHandler(name: RemovableHandler.name)
+        }
+    }
+
+    measure(identifier: identifier + "_handlercontext") {
+        addRemoveBench(iterations: 1000) { channel, handler in
+            channel.pipeline.removeHandler(context: handler.context!)
+        }
+    }
+}

docker/docker-compose.1604.52.yaml

@@ -20,6 +20,9 @@ services:
     environment:
       - MAX_ALLOCS_ALLOWED_1000_addHandlers=47050
       - MAX_ALLOCS_ALLOWED_1000_addHandlers_sync=40050
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlercontext=9035
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlername=19035
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlertype=19035
       - MAX_ALLOCS_ALLOWED_1000_autoReadGetAndSet=32050
       - MAX_ALLOCS_ALLOWED_1000_autoReadGetAndSet_sync=50
       - MAX_ALLOCS_ALLOWED_1000_getHandlers=12050

docker/docker-compose.1604.53.yaml

@@ -20,6 +20,9 @@ services:
     environment:
       - MAX_ALLOCS_ALLOWED_1000_addHandlers=47050
       - MAX_ALLOCS_ALLOWED_1000_addHandlers_sync=40050
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlercontext=9035
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlername=19035
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlertype=19035
       - MAX_ALLOCS_ALLOWED_1000_autoReadGetAndSet=29050
       - MAX_ALLOCS_ALLOWED_1000_autoReadGetAndSet_sync=50
       - MAX_ALLOCS_ALLOWED_1000_getHandlers=12050

docker/docker-compose.1804.50.yaml

@@ -20,6 +20,9 @@ services:
     environment:
       - MAX_ALLOCS_ALLOWED_1000_addHandlers=47050
       - MAX_ALLOCS_ALLOWED_1000_addHandlers_sync=40050
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlercontext=9035
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlername=19035
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlertype=19035
       - MAX_ALLOCS_ALLOWED_1000_autoReadGetAndSet=32050
       - MAX_ALLOCS_ALLOWED_1000_autoReadGetAndSet_sync=50
       - MAX_ALLOCS_ALLOWED_1000_getHandlers=12050

docker/docker-compose.1804.51.yaml

@@ -20,6 +20,9 @@ services:
     environment:
       - MAX_ALLOCS_ALLOWED_1000_addHandlers=47050
       - MAX_ALLOCS_ALLOWED_1000_addHandlers_sync=40050
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlercontext=9035
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlername=19035
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlertype=19035
       - MAX_ALLOCS_ALLOWED_1000_autoReadGetAndSet=32050
       - MAX_ALLOCS_ALLOWED_1000_autoReadGetAndSet_sync=50
       - MAX_ALLOCS_ALLOWED_1000_getHandlers=12050

docker/docker-compose.2004.54.yaml

@@ -20,16 +20,19 @@ services:
     environment:
       - MAX_ALLOCS_ALLOWED_1000_addHandlers=47050
       - MAX_ALLOCS_ALLOWED_1000_addHandlers_sync=40050
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlercontext=9035
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlername=18035  # improvement from 5.3 which was 19035
+      - MAX_ALLOCS_ALLOWED_1000_addRemoveHandlers_handlertype=18035  # improvement from 5.3 which was 19035
       - MAX_ALLOCS_ALLOWED_1000_autoReadGetAndSet=29050
       - MAX_ALLOCS_ALLOWED_1000_autoReadGetAndSet_sync=50
       - MAX_ALLOCS_ALLOWED_1000_getHandlers=12050
       - MAX_ALLOCS_ALLOWED_1000_getHandlers_sync=50
       - MAX_ALLOCS_ALLOWED_1000_reqs_1_conn=30540
       - MAX_ALLOCS_ALLOWED_1000_tcpbootstraps=4100
       - MAX_ALLOCS_ALLOWED_1000_tcpconnections=179050 # regression from 5.3 which was 177010
+      - MAX_ALLOCS_ALLOWED_1000_udp_reqs=16050
       - MAX_ALLOCS_ALLOWED_1000_udpbootstraps=2000
       - MAX_ALLOCS_ALLOWED_1000_udpconnections=102050 # regression from 5.3 which was 101050
-      - MAX_ALLOCS_ALLOWED_1000_udp_reqs=16050
       - MAX_ALLOCS_ALLOWED_1_reqs_1000_conn=468050 # regression from 5.3 which was 465050
       - MAX_ALLOCS_ALLOWED_bytebuffer_lots_of_rw=2100
       - MAX_ALLOCS_ALLOWED_creating_10000_headers=100