From a6a43174839e74d09ed25845a39dfe2879961645 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Fri, 14 Oct 2016 16:07:34 +1100 Subject: [PATCH 01/48] unsubscribe tests + some other fixes --- crossbar/_log_categories.py | 4 +- crossbar/adapter/mqtt/test/test_tx.py | 178 +++++++++++++++++++++++++- crossbar/adapter/mqtt/tx.py | 36 ++++-- 3 files changed, 206 insertions(+), 12 deletions(-) diff --git a/crossbar/_log_categories.py b/crossbar/_log_categories.py index ba7b09128..88f64febf 100644 --- a/crossbar/_log_categories.py +++ b/crossbar/_log_categories.py @@ -73,7 +73,9 @@ "MQ400": "MQTT client '{client_id}' timed out after recieving no full packets for {seconds}", "MQ401": "Protocol violation from '{client_id}', terminating connection: {error}", "MQ402": "Got a packet ('{packet_id}') from '{client_id} that is invalid for a server, terminating connection", - "MQ500": "Error handling a Subscribe from '{client_id}', dropping connection", + "MQ500": "Error handling a Connect, dropping connection", + "MQ501": "Error handling a Subscribe from '{client_id}', dropping connection", + "MQ502": "Error handling an Unsubscribe from '{client_id}', dropping connection", } diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index fa5396613..e29ba61d2 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -39,7 +39,9 @@ from crossbar.adapter.mqtt.protocol import ( MQTTParser, client_packet_handlers, P_CONNACK) from crossbar.adapter.mqtt._events import ( - Connect, ConnectFlags, ConnACK, SubACK, Subscribe, + Connect, ConnectFlags, ConnACK, + SubACK, Subscribe, + Unsubscribe, UnsubACK, SubscriptionTopicRequest ) from crossbar.adapter.mqtt._utils import iterbytes @@ -792,6 +794,85 @@ def process_subscribe(self, event): ).serialise() ) + with LogCapturer("trace") as logs: + for x in iterbytes(data): + p.dataReceived(x) + + sent_logs = logs.get_category("MQ501") + self.assertEqual(len(sent_logs), 1) + self.assertEqual(sent_logs[0]["log_level"], LogLevel.critical) + self.assertEqual(sent_logs[0]["log_failure"].value.args[0], "boom!") + + events = cp.data_received(t.value()) + self.assertEqual(len(events), 1) + self.assertTrue(t.disconnecting) + + # We got the error, we need to flush it so it doesn't make the test + # error + self.flushLoggedErrors() + + +class ConnectHandlingTests(TestCase): + + def test_got_sent_packet(self): + """ + `process_connect` on the handler will get the correct Connect packet. + """ + sessions = {} + got_packets = [] + + class SubHandler(BasicHandler): + def process_connect(self_, event): + got_packets.append(event) + return succeed(0) + + h = SubHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + ) + + for x in iterbytes(data): + p.dataReceived(x) + + self.assertEqual(len(got_packets), 1) + self.assertEqual(got_packets[0].client_id, u"test123") + self.assertEqual(got_packets[0].serialise(), data) + + def test_exception_in_connect_drops_connection(self): + """ + Transient failures (like an exception from handler.process_connect) + will cause the connection it happened on to be dropped. + + Compliance statement MQTT-4.8.0-2 + """ + sessions = {} + + class SubHandler(BasicHandler): + @inlineCallbacks + def process_connect(self, event): + raise Exception("boom!") + + h = SubHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + ) + with LogCapturer("trace") as logs: for x in iterbytes(data): p.dataReceived(x) @@ -801,6 +882,54 @@ def process_subscribe(self, event): self.assertEqual(sent_logs[0]["log_level"], LogLevel.critical) self.assertEqual(sent_logs[0]["log_failure"].value.args[0], "boom!") + events = cp.data_received(t.value()) + self.assertEqual(len(events), 0) + self.assertTrue(t.disconnecting) + + # We got the error, we need to flush it so it doesn't make the test + # error + self.flushLoggedErrors() + + +class UnsubscribeHandlingTests(TestCase): + + + def test_exception_in_connect_drops_connection(self): + """ + Transient failures (like an exception from handler.process_connect) + will cause the connection it happened on to be dropped. + + Compliance statement MQTT-4.8.0-2 + """ + sessions = {} + + class SubHandler(BasicHandler): + def process_unsubscribe(self, event): + raise Exception("boom!") + + h = SubHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + + Unsubscribe(packet_identifier=1234, topics=[u"foo"]).serialise() + ) + + with LogCapturer("trace") as logs: + for x in iterbytes(data): + p.dataReceived(x) + + sent_logs = logs.get_category("MQ502") + self.assertEqual(len(sent_logs), 1) + self.assertEqual(sent_logs[0]["log_level"], LogLevel.critical) + self.assertEqual(sent_logs[0]["log_failure"].value.args[0], "boom!") + events = cp.data_received(t.value()) self.assertEqual(len(events), 1) self.assertTrue(t.disconnecting) @@ -808,3 +937,50 @@ def process_subscribe(self, event): # We got the error, we need to flush it so it doesn't make the test # error self.flushLoggedErrors() + + + def test_unsubscription_gets_unsuback_with_same_id(self): + """ + When an unsubscription is processed, the UnsubACK has the same ID. + Unsubscriptions are always processed. + + Compliance statements MQTT-3.10.4-4, MQTT-3.10.4-5, MQTT-3.12.4-1 + """ + sessions = {} + got_packets = [] + + class SubHandler(BasicHandler): + def process_unsubscribe(self, event): + got_packets.append(event) + return succeed(None) + + h = SubHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + unsub = Unsubscribe(packet_identifier=1234, + topics=[u"foo"]).serialise() + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + unsub + ) + + for x in iterbytes(data): + p.dataReceived(x) + + events = cp.data_received(t.value()) + self.assertEqual(len(events), 2) + self.assertFalse(t.disconnecting) + + # UnsubACK that has the same ID + self.assertIsInstance(events[1], UnsubACK) + self.assertEqual(events[1].packet_identifier, 1234) + + # The unsubscribe handler should have been called + self.assertEqual(len(got_packets), 1) + self.assertEqual(got_packets[0].serialise(), unsub) diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index 0cbf5e4b4..9f7d000da 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -165,7 +165,7 @@ def _flush_saved_messages(self): # Closed connection, we don't want to send messages here if not self.transport.connected: - return + return None while self.session.queued_messages: message = self.session.queued_messages.popleft() @@ -195,8 +195,15 @@ def _handle_data(self, data): client_id=self.session.client_id, packet=event) if isinstance(event, Connect): - - accept_conn = yield self._handler.process_connect(event) + try: + accept_conn = yield self._handler.process_connect(event) + except: + # MQTT-4.8.0-2 - If we get a transient error (like + # connecting raising an exception), we must close the + # connection. + self.log.failure(log_category="MQ500") + self.transport.loseConnection() + return None if accept_conn == 0: # If we have a connection, we should make sure timeouts @@ -214,7 +221,7 @@ def _handle_data(self, data): return_code=2) self._send_packet(connack) self.transport.loseConnection() - return + return None # Use the client ID to control sessions, as per compliance # statement MQTT-3.1.3-2 @@ -251,7 +258,7 @@ def _handle_data(self, data): # No valid return codes, so drop the connection, as per # MQTT-3.2.2-6 self.transport.loseConnection() - return + return None connack = ConnACK(session_present=session_present, return_code=accept_conn) @@ -261,7 +268,7 @@ def _handle_data(self, data): # If we send a CONNACK with a non-0 response code, drop the # connection after sending the CONNACK, as in MQTT-3.2.2-5 self.transport.loseConnection() - return + return None self.log.debug(log_category="MQ200", client_id=event.client_id) continue @@ -274,9 +281,9 @@ def _handle_data(self, data): # subscribing raising an exception), we must close the # connection. self.log.failure( - log_category="MQ500", client_id=self.session.client_id) + log_category="MQ501", client_id=self.session.client_id) self.transport.loseConnection() - return + return None # MQTT-3.8.4-1 - we always need to send back this SubACK, even # if the subscriptions are unsuccessful -- their @@ -288,7 +295,16 @@ def _handle_data(self, data): continue elif isinstance(event, Unsubscribe): - yield self._handler.process_unsubscribe(event) + try: + yield self._handler.process_unsubscribe(event) + except: + # MQTT-4.8.0-2 - If we get a transient error (like + # unsubscribing raising an exception), we must close the + # connection. + self.log.failure( + log_category="MQ502", client_id=self.session.client_id) + self.transport.loseConnection() + return None unsuback = UnsubACK(packet_identifier=event.packet_identifier) self._send_packet(unsuback) continue @@ -336,4 +352,4 @@ def _acked(*args): # Conformance statement MQTT-4.8.0-1: Must close the connection # on a protocol violation. self.transport.loseConnection() - return + return None From 514d97812e60f87364f28f3687ad1f0d756aead7 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 22 Oct 2016 12:22:53 +1100 Subject: [PATCH 02/48] flake8 --- crossbar/adapter/mqtt/test/test_tx.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index e29ba61d2..662c7ae90 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -893,7 +893,6 @@ def process_connect(self, event): class UnsubscribeHandlingTests(TestCase): - def test_exception_in_connect_drops_connection(self): """ Transient failures (like an exception from handler.process_connect) @@ -938,7 +937,6 @@ def process_unsubscribe(self, event): # error self.flushLoggedErrors() - def test_unsubscription_gets_unsuback_with_same_id(self): """ When an unsubscription is processed, the UnsubACK has the same ID. From 35f0479f30085d3dc9c61d87c8d2de1e1a988683 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 22 Oct 2016 13:10:57 +1100 Subject: [PATCH 03/48] qos0 + qos1 tests --- crossbar/_log_categories.py | 3 + crossbar/adapter/mqtt/test/test_tx.py | 199 ++++++++++++++++++++++++++ crossbar/adapter/mqtt/tx.py | 37 +++-- crossbar/adapter/mqtt/wamp.py | 4 +- 4 files changed, 233 insertions(+), 10 deletions(-) diff --git a/crossbar/_log_categories.py b/crossbar/_log_categories.py index 88f64febf..98feef325 100644 --- a/crossbar/_log_categories.py +++ b/crossbar/_log_categories.py @@ -70,12 +70,15 @@ "MQ100": "Got packet from '{client_id}': {packet!r}", "MQ101": "Sent packet to '{client_id}': {packet!r}", "MQ200": "Successful connection from '{client_id}'", + "MQ201": "Received a QoS 0 Publish from '{client_id}'", + "MQ202": "Received a QoS 1 Publish from '{client_id}'", "MQ400": "MQTT client '{client_id}' timed out after recieving no full packets for {seconds}", "MQ401": "Protocol violation from '{client_id}', terminating connection: {error}", "MQ402": "Got a packet ('{packet_id}') from '{client_id} that is invalid for a server, terminating connection", "MQ500": "Error handling a Connect, dropping connection", "MQ501": "Error handling a Subscribe from '{client_id}', dropping connection", "MQ502": "Error handling an Unsubscribe from '{client_id}', dropping connection", + "MQ503": "Error handling a Publish from '{client_id}', dropping connection", } diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index 662c7ae90..d6b7b7888 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -41,6 +41,7 @@ from crossbar.adapter.mqtt._events import ( Connect, ConnectFlags, ConnACK, SubACK, Subscribe, + Publish, PubACK, Unsubscribe, UnsubACK, SubscriptionTopicRequest ) @@ -982,3 +983,201 @@ def process_unsubscribe(self, event): # The unsubscribe handler should have been called self.assertEqual(len(got_packets), 1) self.assertEqual(got_packets[0].serialise(), unsub) + + +class PublishHandlingTests(TestCase): + + def test_qos_0_sends_no_ack(self): + """ + When a QoS 0 Publish packet is recieved, we don't send back a PubACK. + """ + sessions = {} + got_packets = [] + + class PubHandler(BasicHandler): + def process_publish_qos_0(self, event): + got_packets.append(event) + return succeed(None) + + h = PubHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + pub = Publish(duplicate=False, qos_level=0, retain=False, + topic_name=u"foo", packet_identifier=None, + payload=b"bar").serialise() + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + pub + ) + + with LogCapturer("trace") as logs: + for x in iterbytes(data): + p.dataReceived(x) + + events = cp.data_received(t.value()) + self.assertFalse(t.disconnecting) + + # Just the connack, no puback. + self.assertEqual(len(events), 1) + + # The publish handler should have been called + self.assertEqual(len(got_packets), 1) + self.assertEqual(got_packets[0].serialise(), pub) + + # We should get a debug message saying we got the publish + messages = logs.get_category("MQ201") + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0]["publish"].serialise(), pub) + + def test_qos_0_failure_drops_connection(self): + """ + Transient failures (like an exception from + handler.process_publish_qos_0) will cause the connection it happened on + to be dropped. + + Compliance statement MQTT-4.8.0-2 + """ + sessions = {} + + class PubHandler(BasicHandler): + def process_publish_qos_0(self, event): + raise Exception("boom!") + + h = PubHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + + Publish(duplicate=False, qos_level=0, retain=False, + topic_name=u"foo", packet_identifier=None, + payload=b"bar").serialise() + ) + + with LogCapturer("trace") as logs: + for x in iterbytes(data): + p.dataReceived(x) + + sent_logs = logs.get_category("MQ503") + self.assertEqual(len(sent_logs), 1) + self.assertEqual(sent_logs[0]["log_level"], LogLevel.critical) + self.assertEqual(sent_logs[0]["log_failure"].value.args[0], "boom!") + + events = cp.data_received(t.value()) + self.assertEqual(len(events), 1) + self.assertTrue(t.disconnecting) + + # We got the error, we need to flush it so it doesn't make the test + # error + self.flushLoggedErrors() + + def test_qos_1_sends_ack(self): + """ + When a QoS 1 Publish packet is recieved, we send a PubACK with the same + packet identifier as the original Publish. + + Compliance statement MQTT-3.3.4-1 + Spec part 3.4 + """ + sessions = {} + got_packets = [] + + class PubHandler(BasicHandler): + def process_publish_qos_1(self, event): + got_packets.append(event) + return succeed(None) + + h = PubHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + pub = Publish(duplicate=False, qos_level=1, retain=False, + topic_name=u"foo", packet_identifier=2345, + payload=b"bar").serialise() + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + pub + ) + + with LogCapturer("trace") as logs: + for x in iterbytes(data): + p.dataReceived(x) + + events = cp.data_received(t.value()) + self.assertFalse(t.disconnecting) + + # ConnACK + PubACK with the same packet ID + self.assertEqual(len(events), 2) + self.assertEqual(events[1], PubACK(packet_identifier=2345)) + + # The publish handler should have been called + self.assertEqual(len(got_packets), 1) + self.assertEqual(got_packets[0].serialise(), pub) + + # We should get a debug message saying we got the publish + messages = logs.get_category("MQ202") + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0]["publish"].serialise(), pub) + + def test_qos_1_failure_drops_connection(self): + """ + Transient failures (like an exception from + handler.process_publish_qos_1) will cause the connection it happened on + to be dropped. + + Compliance statement MQTT-4.8.0-2 + """ + sessions = {} + + class PubHandler(BasicHandler): + def process_publish_qos_1(self, event): + raise Exception("boom!") + + h = PubHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + + Publish(duplicate=False, qos_level=1, retain=False, + topic_name=u"foo", packet_identifier=2345, + payload=b"bar").serialise() + ) + + with LogCapturer("trace") as logs: + for x in iterbytes(data): + p.dataReceived(x) + + sent_logs = logs.get_category("MQ503") + self.assertEqual(len(sent_logs), 1) + self.assertEqual(sent_logs[0]["log_level"], LogLevel.critical) + self.assertEqual(sent_logs[0]["log_failure"].value.args[0], "boom!") + + events = cp.data_received(t.value()) + self.assertEqual(len(events), 1) + self.assertTrue(t.disconnecting) + + # We got the error, we need to flush it so it doesn't make the test + # error + self.flushLoggedErrors() diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index 9f7d000da..94171e2ce 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -312,18 +312,39 @@ def _handle_data(self, data): elif isinstance(event, Publish): if event.qos_level == 0: # Publish, no acks - self._handler.publish_qos_0(event) + try: + yield self._handler.process_publish_qos_0(event) + except: + # MQTT-4.8.0-2 - If we get a transient error (like + # publishing raising an exception), we must close the + # connection. + self.log.failure(log_category="MQ503", + client_id=self.session.client_id) + self.transport.loseConnection() + return None + + self.log.debug(log_category="MQ201", publish=event, + client_id=self.session.client_id) continue elif event.qos_level == 1: # Publish > PubACK - def _acked(*args): - puback = PubACK( - packet_identifier=event.packet_identifier) - self._send_packet(puback) - - d = self._handler.publish_qos_1(event) - d.addCallback(_acked) + try: + self._handler.process_publish_qos_1(event) + except: + # MQTT-4.8.0-2 - If we get a transient error (like + # publishing raising an exception), we must close the + # connection. + self.log.failure(log_category="MQ503", + client_id=self.session.client_id) + self.transport.loseConnection() + return None + + self.log.debug(log_category="MQ202", publish=event, + client_id=self.session.client_id) + + puback = PubACK(packet_identifier=event.packet_identifier) + self._send_packet(puback) continue elif event.qos_level == 2: diff --git a/crossbar/adapter/mqtt/wamp.py b/crossbar/adapter/mqtt/wamp.py index f608bbc51..5b516d9df 100644 --- a/crossbar/adapter/mqtt/wamp.py +++ b/crossbar/adapter/mqtt/wamp.py @@ -78,10 +78,10 @@ def _publish(self, event, options): return self._wamp_session.publish(event.topic_name, options=options, **payload) - def publish_qos_0(self, event): + def process_publish_qos_0(self, event): return self._publish(event, options=PublishOptions(exclude_me=False)) - def publish_qos_1(self, event): + def process_publish_qos_1(self, event): return self._publish(event, options=PublishOptions(acknowledge=True, exclude_me=False)) From 2541eda0f8e9f0d0e555f435f1203305467cfce1 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 22 Oct 2016 13:39:55 +1100 Subject: [PATCH 04/48] QoS 2 (at the adapter level, not the WAMP level) --- crossbar/_log_categories.py | 8 +- crossbar/adapter/mqtt/_events.py | 98 +++++++++++++++++ crossbar/adapter/mqtt/protocol.py | 12 +- crossbar/adapter/mqtt/test/test_tx.py | 151 +++++++++++++++++++++++++- crossbar/adapter/mqtt/tx.py | 46 +++++++- 5 files changed, 302 insertions(+), 13 deletions(-) diff --git a/crossbar/_log_categories.py b/crossbar/_log_categories.py index 98feef325..3d07dc889 100644 --- a/crossbar/_log_categories.py +++ b/crossbar/_log_categories.py @@ -72,13 +72,17 @@ "MQ200": "Successful connection from '{client_id}'", "MQ201": "Received a QoS 0 Publish from '{client_id}'", "MQ202": "Received a QoS 1 Publish from '{client_id}'", + "MQ203": "Received a QoS 2 Publish from '{client_id}'", "MQ400": "MQTT client '{client_id}' timed out after recieving no full packets for {seconds}", "MQ401": "Protocol violation from '{client_id}', terminating connection: {error}", - "MQ402": "Got a packet ('{packet_id}') from '{client_id} that is invalid for a server, terminating connection", + "MQ402": "Got a packet ('{packet_id}') from '{client_id}' that is invalid for a server, terminating connection", + "MQ403": "Got a Publish packet from '{client_id}' that has both QoS bits set, terminating connection", "MQ500": "Error handling a Connect, dropping connection", "MQ501": "Error handling a Subscribe from '{client_id}', dropping connection", "MQ502": "Error handling an Unsubscribe from '{client_id}', dropping connection", - "MQ503": "Error handling a Publish from '{client_id}', dropping connection", + "MQ503": "Error handling a QoS 0 Publish from '{client_id}', dropping connection", + "MQ504": "Error handling a QoS 1 Publish from '{client_id}', dropping connection", + "MQ505": "Error handling a QoS 2 Publish from '{client_id}', dropping connection", } diff --git a/crossbar/adapter/mqtt/_events.py b/crossbar/adapter/mqtt/_events.py index 50ee59228..358f81222 100644 --- a/crossbar/adapter/mqtt/_events.py +++ b/crossbar/adapter/mqtt/_events.py @@ -160,6 +160,104 @@ def deserialise(cls, flags, data): return cls(packet_identifier=packet_identifier, topics=topics) +@attr.s +class PubCOMP(object): + packet_identifier = attr.ib(validator=instance_of(int)) + + def serialise(self): + """ + Assemble this into an on-wire message. + """ + payload = self._make_payload() + header = build_header(7, (False, False, False, False), len(payload)) + return header + payload + + def _make_payload(self): + """ + Build the payload from its constituent parts. + """ + b = [] + b.append(pack('uint:16', self.packet_identifier).bytes) + return b"".join(b) + + @classmethod + def deserialise(cls, flags, data): + """ + Disassemble from an on-wire message. + """ + if flags != (False, False, False, False): + raise ParseFailure(cls, "Bad flags") + + packet_identifier = data.read('uint:16') + + return cls(packet_identifier) + + +@attr.s +class PubREL(object): + packet_identifier = attr.ib(validator=instance_of(int)) + + def serialise(self): + """ + Assemble this into an on-wire message. + """ + payload = self._make_payload() + header = build_header(6, (False, False, True, False), len(payload)) + return header + payload + + def _make_payload(self): + """ + Build the payload from its constituent parts. + """ + b = [] + b.append(pack('uint:16', self.packet_identifier).bytes) + return b"".join(b) + + @classmethod + def deserialise(cls, flags, data): + """ + Disassemble from an on-wire message. + """ + if flags != (False, False, True, False): + raise ParseFailure(cls, "Bad flags") + + packet_identifier = data.read('uint:16') + + return cls(packet_identifier) + +@attr.s +class PubREC(object): + packet_identifier = attr.ib(validator=instance_of(int)) + + def serialise(self): + """ + Assemble this into an on-wire message. + """ + payload = self._make_payload() + header = build_header(5, (False, False, False, False), len(payload)) + return header + payload + + def _make_payload(self): + """ + Build the payload from its constituent parts. + """ + b = [] + b.append(pack('uint:16', self.packet_identifier).bytes) + return b"".join(b) + + @classmethod + def deserialise(cls, flags, data): + """ + Disassemble from an on-wire message. + """ + if flags != (False, False, False, False): + raise ParseFailure(cls, "Bad flags") + + packet_identifier = data.read('uint:16') + + return cls(packet_identifier) + + @attr.s class PubACK(object): packet_identifier = attr.ib(validator=instance_of(int)) diff --git a/crossbar/adapter/mqtt/protocol.py b/crossbar/adapter/mqtt/protocol.py index 37b69ac5d..34d2f53c6 100644 --- a/crossbar/adapter/mqtt/protocol.py +++ b/crossbar/adapter/mqtt/protocol.py @@ -36,17 +36,13 @@ Subscribe, SubACK, Unsubscribe, UnsubACK, Publish, PubACK, + PubREC, PubREL, PubCOMP, PingREQ, PingRESP, ) import bitstring __all__ = [ - "Connect", "ConnACK", - "Subscribe", "SubACK", - "Unsubscribe", "UnsubACK", - "Publish", "PubACK", - "PingREQ", "PingRESP", "MQTTParser", ] @@ -65,6 +61,9 @@ class _NeedMoreData(Exception): P_CONNACK = 2 P_PUBLISH = 3 P_PUBACK = 4 +P_PUBREC = 5 +P_PUBREL = 6 +P_PUBCOMP = 7 P_SUBSCRIBE = 8 P_SUBACK = 9 P_UNSUBSCRIBE = 10 @@ -78,6 +77,7 @@ class _NeedMoreData(Exception): P_SUBSCRIBE: Subscribe, P_UNSUBSCRIBE: Unsubscribe, P_PINGREQ: PingREQ, + P_PUBREL: PubREL, } client_packet_handlers = { @@ -87,6 +87,8 @@ class _NeedMoreData(Exception): P_SUBACK: SubACK, P_UNSUBACK: UnsubACK, P_PINGRESP: PingRESP, + P_PUBREC: PubREC, + P_PUBCOMP: PubCOMP, } diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index d6b7b7888..e2c8c58e5 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -41,7 +41,7 @@ from crossbar.adapter.mqtt._events import ( Connect, ConnectFlags, ConnACK, SubACK, Subscribe, - Publish, PubACK, + Publish, PubACK, PubREC, PubREL, PubCOMP, Unsubscribe, UnsubACK, SubscriptionTopicRequest ) @@ -631,6 +631,37 @@ def test_lose_conn_on_unimplemented_packet(self): self.assertTrue(t.disconnecting) + def test_lose_conn_on_reserved_qos3(self): + """ + If we get, somehow, a QoS "3" Publish (one with both QoS bits set to + 3), we will drop the connection. + + Compliance statement: MQTT-3.3.1-4 + """ + sessions = {} + + h = BasicHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + + p.makeConnection(t) + + conn = Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=False)) + pub = Publish(duplicate=False, qos_level=3, retain=False, + topic_name=u"foo", packet_identifier=2345, + payload=b"bar") + + with LogCapturer("trace") as logs: + p._handle_events([conn, pub]) + + sent_logs = logs.get_category("MQ403") + self.assertEqual(len(sent_logs), 1) + self.assertEqual(sent_logs[0]["log_level"], LogLevel.error) + + self.assertTrue(t.disconnecting) + class NonZeroConnACKTests(object): @@ -1169,7 +1200,123 @@ def process_publish_qos_1(self, event): for x in iterbytes(data): p.dataReceived(x) - sent_logs = logs.get_category("MQ503") + sent_logs = logs.get_category("MQ504") + self.assertEqual(len(sent_logs), 1) + self.assertEqual(sent_logs[0]["log_level"], LogLevel.critical) + self.assertEqual(sent_logs[0]["log_failure"].value.args[0], "boom!") + + events = cp.data_received(t.value()) + self.assertEqual(len(events), 1) + self.assertTrue(t.disconnecting) + + # We got the error, we need to flush it so it doesn't make the test + # error + self.flushLoggedErrors() + + def test_qos_2_sends_ack(self): + """ + When a QoS 2 Publish packet is recieved, we send a PubREC with the same + packet identifier as the original Publish, wait for a PubREL, and then + send a PubCOMP. + + Compliance statement MQTT-4.3.3-2 + Spec part 3.4, 4.3.3 + """ + sessions = {} + got_packets = [] + + class PubHandler(BasicHandler): + def process_publish_qos_2(self, event): + got_packets.append(event) + return succeed(None) + + h = PubHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + pub = Publish(duplicate=False, qos_level=2, retain=False, + topic_name=u"foo", packet_identifier=2345, + payload=b"bar").serialise() + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + pub + ) + + with LogCapturer("trace") as logs: + for x in iterbytes(data): + p.dataReceived(x) + + events = cp.data_received(t.value()) + self.assertFalse(t.disconnecting) + + # ConnACK + PubREC with the same packet ID + self.assertEqual(len(events), 2) + self.assertEqual(events[1], PubREC(packet_identifier=2345)) + + # The publish handler should have been called + self.assertEqual(len(got_packets), 1) + self.assertEqual(got_packets[0].serialise(), pub) + + # We should get a debug message saying we got the publish + messages = logs.get_category("MQ203") + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0]["publish"].serialise(), pub) + + # Clear the client transport + t.clear() + + # Now we send the PubREL + pubrel = PubREL(packet_identifier=2345) + for x in iterbytes(pubrel.serialise()): + p.dataReceived(x) + + events = cp.data_received(t.value()) + self.assertFalse(t.disconnecting) + + # We should get a PubCOMP in response + self.assertEqual(len(events), 1) + self.assertEqual(events[0], PubCOMP(packet_identifier=2345)) + + def test_qos_2_failure_drops_connection(self): + """ + Transient failures (like an exception from + handler.process_publish_qos_2) will cause the connection it happened on + to be dropped. + + Compliance statement MQTT-4.8.0-2 + """ + sessions = {} + + class PubHandler(BasicHandler): + def process_publish_qos_2(self, event): + raise Exception("boom!") + + h = PubHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + + Publish(duplicate=False, qos_level=2, retain=False, + topic_name=u"foo", packet_identifier=2345, + payload=b"bar").serialise() + ) + + with LogCapturer("trace") as logs: + for x in iterbytes(data): + p.dataReceived(x) + + sent_logs = logs.get_category("MQ505") self.assertEqual(len(sent_logs), 1) self.assertEqual(sent_logs[0]["log_level"], LogLevel.critical) self.assertEqual(sent_logs[0]["log_failure"].value.args[0], "boom!") diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index 94171e2ce..e6b018c5b 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -39,10 +39,12 @@ from .protocol import ( MQTTParser, Failure, +) +from ._events import ( Connect, ConnACK, Subscribe, SubACK, Unsubscribe, UnsubACK, - Publish, PubACK, + Publish, PubACK, PubREC, PubREL, PubCOMP, PingREQ, PingRESP, ) @@ -180,7 +182,6 @@ def _handle(self, data): except Exception: raise - @inlineCallbacks def _handle_data(self, data): events = self._mqtt.data_received(data) @@ -190,6 +191,11 @@ def _handle_data(self, data): # alive, reset the timeout. self._reset_timeout() + return self._handle_events(events) + + @inlineCallbacks + def _handle_events(self, events): + for event in events: self.log.trace(log_category="MQ100", conn_id=self._connection_id, client_id=self.session.client_id, packet=event) @@ -335,7 +341,7 @@ def _handle_data(self, data): # MQTT-4.8.0-2 - If we get a transient error (like # publishing raising an exception), we must close the # connection. - self.log.failure(log_category="MQ503", + self.log.failure(log_category="MQ504", client_id=self.session.client_id) self.transport.loseConnection() return None @@ -353,13 +359,45 @@ def _handle_data(self, data): # add to set, send pubrec here -- in the branching loop, # handle pubrel + pubcomp - raise ValueError("Dunno about this yet.") + try: + self._handler.process_publish_qos_2(event) + except: + # MQTT-4.8.0-2 - If we get a transient error (like + # publishing raising an exception), we must close the + # connection. + self.log.failure(log_category="MQ505", + client_id=self.session.client_id) + self.transport.loseConnection() + return None + + self.log.debug(log_category="MQ203", publish=event, + client_id=self.session.client_id) + + pubrec = PubREC(packet_identifier=event.packet_identifier) + self._send_packet(pubrec) + continue + + else: + # MQTT-3.3.1-4 - We got a QoS "3" (both QoS bits set) + # packet -- something the spec does not allow! Nor our + # events implementation (it will be caught before it gets + # here), but the tests do some trickery to cover this + # case :) + self.log.error(log_category="MQ403", + client_id=self.session.client_id) + self.transport.loseConnection() + return elif isinstance(event, PingREQ): resp = PingRESP() self._send_packet(resp) continue + elif isinstance(event, PubREL): + resp = PubCOMP(packet_identifier=event.packet_identifier) + self._send_packet(resp) + continue + else: if isinstance(event, Failure): self.log.error( From 62cea5cb3efdedad6c24b5ad05288b27bf4f64e9 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 22 Oct 2016 14:25:40 +1100 Subject: [PATCH 05/48] add MQTT to unisocket --- crossbar/router/test/test_unisocket.py | 52 ++++++++++++++++++++++++-- crossbar/router/unisocket.py | 14 ++++++- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/crossbar/router/test/test_unisocket.py b/crossbar/router/test/test_unisocket.py index 8a72f315e..4749c9188 100644 --- a/crossbar/router/test/test_unisocket.py +++ b/crossbar/router/test/test_unisocket.py @@ -110,8 +110,8 @@ def test_web_with_no_factory(self): def test_invalid_status_line(self): """ - Not speaking RawSocket but also not speaking a type of HTTP will cause - the connection to be dropped. + Not speaking RawSocket or MQTT but also not speaking a type of HTTP + will cause the connection to be dropped. """ t = StringTransport() @@ -185,7 +185,7 @@ def dataReceived(self, data): def test_websocket_with_no_map(self): """ - A web request that matches no WebSocket + A web request that matches no WebSocket path will go to HTTP/1.1. """ t = StringTransport() @@ -202,3 +202,49 @@ def test_websocket_with_no_map(self): self.assertFalse(t.connected) self.assertEqual(t.value(), b"") + + def test_mqtt_with_no_factory(self): + """ + Trying to speak MQTT with no MQTT factory configured will + drop the connection. + """ + t = StringTransport() + + f = UniSocketServerFactory() + p = f.buildProtocol(None) + + p.makeConnection(t) + t.protocol = p + + self.assertTrue(t.connected) + p.dataReceived(b'\x100000000') + + self.assertFalse(t.connected) + + def test_mqtt_with_factory(self): + """ + Speaking MQTT when the connection is made will make UniSocket + create a new MQTT protocol and send the data to it. + """ + t = StringTransport() + + class MyFakeMQTT(Protocol): + """ + A fake MQTT factory which just echos data back. + """ + def dataReceived(self, data): + self.transport.write(data) + + fake_mqtt = Factory.forProtocol(MyFakeMQTT) + f = UniSocketServerFactory(mqtt_factory=fake_mqtt) + p = f.buildProtocol(None) + + p.makeConnection(t) + t.protocol = p + + self.assertTrue(t.connected) + p.dataReceived(b'\x100000000') + p.dataReceived(b'moredata') + + self.assertTrue(t.connected) + self.assertEqual(t.value(), b'\x100000000moredata') diff --git a/crossbar/router/unisocket.py b/crossbar/router/unisocket.py index 71fffdb11..e502db673 100644 --- a/crossbar/router/unisocket.py +++ b/crossbar/router/unisocket.py @@ -73,6 +73,17 @@ def dataReceived(self, data): self._proto.transport = self.transport self._proto.connectionMade() self._proto.dataReceived(data) + elif data[0:1] == b'\x10': + # switch to MQTT + if not self._factory._mqtt_factory: + self.log.warn('client wants to talk MQTT, but we have no factory configured for that') + self.transport.loseConnection() + else: + self.log.debug('switching to MQTT') + self._proto = self._factory._mqtt_factory.buildProtocol(self._addr) + self._proto.transport = self.transport + self._proto.connectionMade() + self._proto.dataReceived(data) else: # switch to HTTP, further subswitching to WebSocket (from Autobahn, like a WebSocketServerFactory) # or Web (from Twisted Web, like a Site). the subswitching is based on HTTP Request-URI. @@ -160,12 +171,13 @@ class UniSocketServerFactory(Factory): noisy = False - def __init__(self, web_factory=None, websocket_factory_map=None, rawsocket_factory=None): + def __init__(self, web_factory=None, websocket_factory_map=None, rawsocket_factory=None, mqtt_factory=None): """ """ self._web_factory = web_factory self._websocket_factory_map = websocket_factory_map self._rawsocket_factory = rawsocket_factory + self._mqtt_factory = mqtt_factory def buildProtocol(self, addr): proto = UniSocketServerProtocol(self, addr) From 581832ff77c6ba08b3b627a517ffd1666845d94f Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 22 Oct 2016 14:27:31 +1100 Subject: [PATCH 06/48] add to checkconfig --- crossbar/common/checkconfig.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crossbar/common/checkconfig.py b/crossbar/common/checkconfig.py index 4fe5033f0..9d62f5301 100644 --- a/crossbar/common/checkconfig.py +++ b/crossbar/common/checkconfig.py @@ -1595,6 +1595,7 @@ def check_listening_transport_universal(transport): 'endpoint', 'rawsocket', 'websocket', + 'mqtt', 'web', ]: raise InvalidConfigException("encountered unknown attribute '{}' in Universal transport configuration".format(k)) @@ -1621,6 +1622,9 @@ def check_listening_transport_universal(transport): for path in paths: check_listening_transport_websocket(transport['websocket'][path], with_endpoint=False) + if 'mqtt' in transport: + check_listening_transport_mqtt(transport['mqtt'], with_endpoint=False) + if 'web' in transport: check_listening_transport_web(transport['web'], with_endpoint=False) From 10621801eab4a32b40c3cbbbe9a528d997832051 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 22 Oct 2016 14:29:20 +1100 Subject: [PATCH 07/48] add universal mqtt to router config --- crossbar/worker/router.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crossbar/worker/router.py b/crossbar/worker/router.py index 0ac8e1776..2a9194423 100644 --- a/crossbar/worker/router.py +++ b/crossbar/worker/router.py @@ -837,6 +837,13 @@ def start_router_transport(self, id, config, details=None): else: rawsocket_factory = None + if 'mqtt' in config: + mqtt_factory = WampMQTTServerFactory( + self._router_session_factory, config, self._reactor) + mqtt_factory.noisy = False + else: + mqtt_factory = None + if 'websocket' in config: websocket_factory_map = {} for websocket_url_first_component, websocket_config in config['websocket'].items(): @@ -847,7 +854,7 @@ def start_router_transport(self, id, config, details=None): else: websocket_factory_map = None - transport_factory = UniSocketServerFactory(web_factory, websocket_factory_map, rawsocket_factory) + transport_factory = UniSocketServerFactory(web_factory, websocket_factory_map, rawsocket_factory, mqtt_factory) # Unknown transport type # From cd65598a0561c6cef3cd3f237f04e9b11ac4c36e Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 22 Oct 2016 14:50:40 +1100 Subject: [PATCH 08/48] handle pinned/unpinned dependencies in Travis, since the abtrunk/twtrunk was overwritten anyway --- .gitignore | 1 + .travis.yml | 18 ++++++------------ tox.ini | 10 ++-------- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 64c2ccdba..2faab08cb 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ node_modules *.swp *.ropeproject docs/_build +.coverage.* diff --git a/.travis.yml b/.travis.yml index ef53043e6..77575703c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,13 +8,11 @@ install: env: - TOX_ENV=flake8 - - TOX_ENV=py27-twtrunk-abtrunk,coverage-report - - TOX_ENV=py27-twcurrent-abtrunk,coverage-report - #- TOX_ENV=py27-twcurrent-abcurrent + - TOX_ENV=py27-unpinned,coverage-report + - TOX_ENV=py27-pinned,coverage-report - - TOX_ENV=py34-twtrunk-abtrunk,coverage-report - - TOX_ENV=py34-twcurrent-abtrunk,coverage-report - #- TOX_ENV=py34-twcurrent-abcurrent + - TOX_ENV=py34-unpinned,coverage-report + - TOX_ENV=py34-pinned,coverage-report # Travis needs to update it's PyPy #- TOX_ENV=pypy-twtrunk-abtrunk @@ -41,12 +39,8 @@ matrix: include: - python: 3.5 env: - - TOX_ENV=py35-twtrunk-abtrunk,coverage-report + - TOX_ENV=py35-unpinned,coverage-report - python: 3.5 env: - - TOX_ENV=py35-twcurrent-abtrunk,coverage-report - -# - python: 3.5 -# env: -# - TOX_ENV=py35-twcurrent-abcurrent + - TOX_ENV=py35-pinned,coverage-report diff --git a/tox.ini b/tox.ini index fc64adb9e..f532c1781 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = flake8, {py27,pypy,py33,py34,py35}-{twtrunk,twcurrent}-{abtrunk,abcurrent} +envlist = flake8, {py27,pypy,py33,py34,py35}-{pinned,unpinned} [flake8] exclude = crossbar/worker/test/examples/syntaxerror.py @@ -39,11 +39,6 @@ deps = #pypy: pynacl<1.0 mock coverage - abcurrent: autobahn - abtrunk: https://github.com/crossbario/txaio/archive/master.zip - abtrunk: https://github.com/crossbario/autobahn-python/archive/master.zip - twcurrent: twisted - twtrunk: https://github.com/twisted/twisted/archive/trunk.zip commands = sh -c "which python" sh -c "which coverage" @@ -52,8 +47,7 @@ commands = python -c "import setuptools; print('setuptools-%s' % setuptools.__version__)" coverage --version {envbindir}/trial --version - #{py27,py33,py34,py35,pypy}: pip install . - {py27,py33,py34,py35,pypy}: pip install --ignore-installed --require-hashes -r {toxinidir}/requirements.txt + pinned: pip install --ignore-installed --require-hashes -r {toxinidir}/requirements.txt coverage run --rcfile={toxinidir}/.coveragerc -m twisted.trial {posargs:crossbar} setenv = COVERAGE_PROCESS_START = {toxinidir}/.coveragerc From f2c60ef8783c634f48490018131383880590e0b2 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 22 Oct 2016 14:51:15 +1100 Subject: [PATCH 09/48] fix tests on py2 --- crossbar/adapter/mqtt/tx.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index e6b018c5b..a6f8ce15d 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -209,7 +209,7 @@ def _handle_events(self, events): # connection. self.log.failure(log_category="MQ500") self.transport.loseConnection() - return None + returnValue(None) if accept_conn == 0: # If we have a connection, we should make sure timeouts @@ -227,7 +227,7 @@ def _handle_events(self, events): return_code=2) self._send_packet(connack) self.transport.loseConnection() - return None + returnValue(None) # Use the client ID to control sessions, as per compliance # statement MQTT-3.1.3-2 @@ -264,7 +264,7 @@ def _handle_events(self, events): # No valid return codes, so drop the connection, as per # MQTT-3.2.2-6 self.transport.loseConnection() - return None + returnValue(None) connack = ConnACK(session_present=session_present, return_code=accept_conn) @@ -274,7 +274,7 @@ def _handle_events(self, events): # If we send a CONNACK with a non-0 response code, drop the # connection after sending the CONNACK, as in MQTT-3.2.2-5 self.transport.loseConnection() - return None + returnValue(None) self.log.debug(log_category="MQ200", client_id=event.client_id) continue @@ -289,7 +289,7 @@ def _handle_events(self, events): self.log.failure( log_category="MQ501", client_id=self.session.client_id) self.transport.loseConnection() - return None + returnValue(None) # MQTT-3.8.4-1 - we always need to send back this SubACK, even # if the subscriptions are unsuccessful -- their @@ -310,7 +310,7 @@ def _handle_events(self, events): self.log.failure( log_category="MQ502", client_id=self.session.client_id) self.transport.loseConnection() - return None + returnValue(None) unsuback = UnsubACK(packet_identifier=event.packet_identifier) self._send_packet(unsuback) continue @@ -327,7 +327,7 @@ def _handle_events(self, events): self.log.failure(log_category="MQ503", client_id=self.session.client_id) self.transport.loseConnection() - return None + returnValue(None) self.log.debug(log_category="MQ201", publish=event, client_id=self.session.client_id) @@ -344,7 +344,7 @@ def _handle_events(self, events): self.log.failure(log_category="MQ504", client_id=self.session.client_id) self.transport.loseConnection() - return None + returnValue(None) self.log.debug(log_category="MQ202", publish=event, client_id=self.session.client_id) @@ -368,7 +368,7 @@ def _handle_events(self, events): self.log.failure(log_category="MQ505", client_id=self.session.client_id) self.transport.loseConnection() - return None + returnValue(None) self.log.debug(log_category="MQ203", publish=event, client_id=self.session.client_id) @@ -411,4 +411,4 @@ def _handle_events(self, events): # Conformance statement MQTT-4.8.0-1: Must close the connection # on a protocol violation. self.transport.loseConnection() - return None + returnValue(None) From 2d17afbe26ddc4244a1c8e4b6e8e82bf5e4a88d5 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 22 Oct 2016 16:11:41 +1100 Subject: [PATCH 10/48] packets ids, in-flight qos 1 --- crossbar/adapter/mqtt/test/test_tx.py | 192 +++++++++++++++++++++++++- crossbar/adapter/mqtt/tx.py | 37 ++++- 2 files changed, 226 insertions(+), 3 deletions(-) diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index e2c8c58e5..5e1d13c80 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -35,7 +35,7 @@ from functools import partial from binascii import unhexlify -from crossbar.adapter.mqtt.tx import MQTTServerTwistedProtocol +from crossbar.adapter.mqtt.tx import MQTTServerTwistedProtocol, AwaitingACK from crossbar.adapter.mqtt.protocol import ( MQTTParser, client_packet_handlers, P_CONNACK) from crossbar.adapter.mqtt._events import ( @@ -1328,3 +1328,193 @@ def process_publish_qos_2(self, event): # We got the error, we need to flush it so it doesn't make the test # error self.flushLoggedErrors() + + +class SendPublishTests(TestCase): + """ + Tests for the WAMP layer sending messages to MQTT clients. + """ + + def test_qos_0_queues_message(self): + """ + The WAMP layer calling send_publish will queue a message up for + sending, and send it next time it has a chance. + """ + sessions = {} + got_packets = [] + + h = BasicHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + ) + + for x in iterbytes(data): + p.dataReceived(x) + + # No queued messages + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + + # Connect has happened + events = cp.data_received(t.value()) + t.clear() + self.assertFalse(t.disconnecting) + self.assertIsInstance(events[0], ConnACK) + + # WAMP layer calls send_publish + p.send_publish(u"hello", 0, b'some bytes') + + # Nothing should have been sent yet, it is queued + self.assertEqual(t.value(), b'') + self.assertEqual(len(sessions[u"test123"].queued_messages), 1) + + # Advance the clock + r.advance(0.1) + + # We should now get the sent Publish + events = cp.data_received(t.value()) + self.assertEqual(len(events), 1) + self.assertEqual( + events[0], + Publish(duplicate=False, qos_level=0, retain=False, + packet_identifier=None, topic_name=u"hello", + payload=b"some bytes")) + + def test_qos_1_queues_message(self): + """ + The WAMP layer calling send_publish will queue a message up for + sending, and send it next time it has a chance. + """ + sessions = {} + got_packets = [] + + h = BasicHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + ) + + for x in iterbytes(data): + p.dataReceived(x) + + # No queued messages + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + + # Make the packet ID be deterministic + sessions[u"test123"].get_packet_id = lambda: 4567 + + # Connect has happened + events = cp.data_received(t.value()) + t.clear() + self.assertFalse(t.disconnecting) + self.assertIsInstance(events[0], ConnACK) + + # WAMP layer calls send_publish, with QoS 1 + p.send_publish(u"hello", 1, b'some bytes') + + # Nothing should have been sent yet, it is queued + self.assertEqual(t.value(), b'') + self.assertEqual(len(sessions[u"test123"].queued_messages), 1) + + # Advance the clock + r.advance(0.1) + + # We should now get the sent Publish + events = cp.data_received(t.value()) + t.clear() + self.assertEqual(len(events), 1) + self.assertEqual( + events[0], + Publish(duplicate=False, qos_level=1, retain=False, + packet_identifier=4567, topic_name=u"hello", + payload=b"some bytes")) + + # Server is still awaiting the client's response + self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) + self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], + AwaitingACK(qos=1, stage=0)) + + # We send the PubACK, which we don't get a response to + puback = PubACK(packet_identifier=4567) + + for x in iterbytes(puback.serialise()): + p.dataReceived(x) + + events = cp.data_received(t.value()) + self.assertEqual(len(events), 0) + + # It is no longer queued + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 0) + self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) + + def test_qos_2_queues_message(self): + """ + The WAMP layer calling send_publish will queue a message up for + sending, and send it next time it has a chance. + """ + sessions = {} + got_packets = [] + + h = BasicHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + ) + + for x in iterbytes(data): + p.dataReceived(x) + + # No queued messages + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + + # Make the packet ID be deterministic + sessions[u"test123"].get_packet_id = lambda: 4567 + + # Connect has happened + events = cp.data_received(t.value()) + t.clear() + self.assertFalse(t.disconnecting) + self.assertIsInstance(events[0], ConnACK) + + # WAMP layer calls send_publish, with QoS 2 + p.send_publish(u"hello", 2, b'some bytes') + + # Nothing should have been sent yet, it is queued + self.assertEqual(t.value(), b'') + self.assertEqual(len(sessions[u"test123"].queued_messages), 1) + + # Advance the clock + r.advance(0.1) + + # We should now get the sent Publish + events = cp.data_received(t.value()) + self.assertEqual(len(events), 1) + self.assertEqual( + events[0], + Publish(duplicate=False, qos_level=2, retain=False, + packet_identifier=4567, topic_name=u"hello", + payload=b"some bytes")) + + raise ValueError("Need to handle PubREC/PubREL/PubCOMP") diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index a6f8ce15d..5e55d6e9d 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -33,6 +33,7 @@ import attr import collections +from random import randint from itertools import count from txaio import make_logger @@ -52,6 +53,7 @@ from twisted.internet.defer import inlineCallbacks, returnValue _ids = count() +_SIXTEEN_BIT_MAX = 65535 @attr.s @@ -63,6 +65,14 @@ class Session(object): subscriptions = attr.ib(default=attr.Factory(dict)) connected = attr.ib(default=False) survives = attr.ib(default=False) + _in_flight_packet_ids = attr.ib(default=attr.Factory(set)) + _publishes_awaiting_ack = attr.ib(default=attr.Factory(dict)) + + def get_packet_id(self): + x = 0 + while x == 0 or x in self._in_flight_packet_ids: + x = randint(1, _SIXTEEN_BIT_MAX) + return x @attr.s @@ -73,6 +83,13 @@ class Message(object): qos = attr.ib() +@attr.s +class AwaitingACK(object): + + qos = attr.ib() + stage = attr.ib() + + class MQTTServerTwistedProtocol(Protocol): log = make_logger() @@ -140,12 +157,28 @@ def _send_publish(self, topic, qos, body): payload=body) elif qos == 1: - packet_id = self._get_packet_id() + packet_id = self.session.get_packet_id() publish = Publish(duplicate=False, qos_level=qos, retain=False, packet_identifier=packet_id, topic_name=topic, payload=body) - self._waiting_for_ack[packet_id] = (1, 0) + waiting_ack = AwaitingACK(qos=1, stage=0) + self.session._publishes_awaiting_ack[packet_id] = waiting_ack + self.session._in_flight_packet_ids.add(packet_id) + + elif qos == 2: + + packet_id = self.session.get_packet_id() + publish = Publish(duplicate=False, qos_level=qos, retain=False, + packet_identifier=packet_id, topic_name=topic, + payload=body) + + waiting_ack = AwaitingACK(qos=2, stage=0) + self.session._publishes_awaiting_ack[packet_id] = waiting_ack + self.session._in_flight_packet_ids.add(packet_id) + + else: + raise ValueError("QoS must be [0, 1, 2]") self._send_packet(publish) From b43f1a9bfd2abfa385cf811f9294d551ab975966 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 22 Oct 2016 17:46:15 +1100 Subject: [PATCH 11/48] branches and starting on qos 2 coverage --- crossbar/_log_categories.py | 1 + crossbar/adapter/mqtt/protocol.py | 3 + crossbar/adapter/mqtt/test/test_tx.py | 105 +++++++++++++++++++++++--- crossbar/adapter/mqtt/tx.py | 17 ++++- 4 files changed, 112 insertions(+), 14 deletions(-) diff --git a/crossbar/_log_categories.py b/crossbar/_log_categories.py index 3d07dc889..667a95cb2 100644 --- a/crossbar/_log_categories.py +++ b/crossbar/_log_categories.py @@ -73,6 +73,7 @@ "MQ201": "Received a QoS 0 Publish from '{client_id}'", "MQ202": "Received a QoS 1 Publish from '{client_id}'", "MQ203": "Received a QoS 2 Publish from '{client_id}'", + "MQ300": "Got a PubACK for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", "MQ400": "MQTT client '{client_id}' timed out after recieving no full packets for {seconds}", "MQ401": "Protocol violation from '{client_id}', terminating connection: {error}", "MQ402": "Got a packet ('{packet_id}') from '{client_id}' that is invalid for a server, terminating connection", diff --git a/crossbar/adapter/mqtt/protocol.py b/crossbar/adapter/mqtt/protocol.py index 34d2f53c6..7825561d1 100644 --- a/crossbar/adapter/mqtt/protocol.py +++ b/crossbar/adapter/mqtt/protocol.py @@ -74,10 +74,13 @@ class _NeedMoreData(Exception): server_packet_handlers = { P_CONNECT: Connect, P_PUBLISH: Publish, + P_PUBACK: PubACK, P_SUBSCRIBE: Subscribe, P_UNSUBSCRIBE: Unsubscribe, P_PINGREQ: PingREQ, P_PUBREL: PubREL, + P_PUBREC: PubREC, + P_PUBCOMP: PubCOMP, } client_packet_handlers = { diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index 5e1d13c80..78421589a 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -1434,20 +1434,19 @@ def test_qos_1_queues_message(self): r.advance(0.1) # We should now get the sent Publish + expected_publish = Publish( + duplicate=False, qos_level=1, retain=False, packet_identifier=4567, + topic_name=u"hello", payload=b"some bytes") events = cp.data_received(t.value()) t.clear() self.assertEqual(len(events), 1) - self.assertEqual( - events[0], - Publish(duplicate=False, qos_level=1, retain=False, - packet_identifier=4567, topic_name=u"hello", - payload=b"some bytes")) + self.assertEqual(events[0], expected_publish) # Server is still awaiting the client's response self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], - AwaitingACK(qos=1, stage=0)) + AwaitingACK(qos=1, stage=0, message=expected_publish)) # We send the PubACK, which we don't get a response to puback = PubACK(packet_identifier=4567) @@ -1461,6 +1460,56 @@ def test_qos_1_queues_message(self): # It is no longer queued self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 0) self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertFalse(t.disconnecting) + + def test_qos_1_unassociated_puback_warns(self): + """ + On receiving a PubACK which does not reference any known existing + Publish the server has sent, it will drop it and raise a warning. + + XXX: Spec is unclear if this is the proper behaviour! + """ + sessions = {} + got_packets = [] + + h = BasicHandler() + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(h, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + ) + + for x in iterbytes(data): + p.dataReceived(x) + + # Connect has happened + events = cp.data_received(t.value()) + t.clear() + self.assertFalse(t.disconnecting) + self.assertIsInstance(events[0], ConnACK) + + # We send the PubACK, which we don't get a response to (and also + # doesn't refer to any Publish the client was sent) + puback = PubACK(packet_identifier=4567) + + with LogCapturer("trace") as logs: + for x in iterbytes(puback.serialise()): + p.dataReceived(x) + + events = cp.data_received(t.value()) + self.assertEqual(len(events), 0) + + # A warning is raised + sent_logs = logs.get_category("MQ300") + self.assertEqual(len(sent_logs), 1) + self.assertEqual(sent_logs[0]["log_level"], LogLevel.warn) + self.assertEqual(sent_logs[0]["pub_id"], 4567) def test_qos_2_queues_message(self): """ @@ -1509,12 +1558,44 @@ def test_qos_2_queues_message(self): r.advance(0.1) # We should now get the sent Publish + expected_publish = Publish(duplicate=False, qos_level=2, retain=False, + packet_identifier=4567, topic_name=u"hello", + payload=b"some bytes") events = cp.data_received(t.value()) + t.clear() self.assertEqual(len(events), 1) - self.assertEqual( - events[0], - Publish(duplicate=False, qos_level=2, retain=False, - packet_identifier=4567, topic_name=u"hello", - payload=b"some bytes")) + self.assertEqual(events[0], expected_publish) + + # Server is still awaiting the client's response + self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) + self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], + AwaitingACK(qos=2, stage=0, message=expected_publish)) + + # We send the PubREC, which we should get a PubREL back with + pubrec = PubREC(packet_identifier=4567) - raise ValueError("Need to handle PubREC/PubREL/PubCOMP") + for x in iterbytes(pubrec.serialise()): + p.dataReceived(x) + + events = cp.data_received(t.value()) + t.clear() + self.assertEqual(len(events), 1) + self.assertEqual(events[0], PubREL(packet_identifier=4567)) + + # Server is still awaiting the client's response, it is not complete + self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) + self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], + AwaitingACK(qos=2, stage=1, message=expected_publish)) + + # We send the PubCOMP, which has no response + pubcomp = PubCOMP(packet_identifier=4567) + + for x in iterbytes(pubcomp.serialise()): + p.dataReceived(x) + + # It is no longer queued + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 0) + self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertFalse(t.disconnecting) diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index 5e55d6e9d..b71176dbb 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -88,6 +88,7 @@ class AwaitingACK(object): qos = attr.ib() stage = attr.ib() + message = attr.ib() class MQTTServerTwistedProtocol(Protocol): @@ -162,7 +163,7 @@ def _send_publish(self, topic, qos, body): packet_identifier=packet_id, topic_name=topic, payload=body) - waiting_ack = AwaitingACK(qos=1, stage=0) + waiting_ack = AwaitingACK(qos=1, stage=0, message=publish) self.session._publishes_awaiting_ack[packet_id] = waiting_ack self.session._in_flight_packet_ids.add(packet_id) @@ -173,7 +174,7 @@ def _send_publish(self, topic, qos, body): packet_identifier=packet_id, topic_name=topic, payload=body) - waiting_ack = AwaitingACK(qos=2, stage=0) + waiting_ack = AwaitingACK(qos=2, stage=0, message=publish) self.session._publishes_awaiting_ack[packet_id] = waiting_ack self.session._in_flight_packet_ids.add(packet_id) @@ -426,6 +427,18 @@ def _handle_events(self, events): self._send_packet(resp) continue + elif isinstance(event, PubACK): + + if event.packet_identifier in self.session._publishes_awaiting_ack: + # MQTT-4.3.2-1: Only acknowledge when it has been PubACK'd + del self.session._publishes_awaiting_ack[event.packet_identifier] + self.session._in_flight_packet_ids.remove(event.packet_identifier) + + else: + self.log.warn( + log_category="MQ300", client_id=self.session.client_id, + pub_id=event.packet_identifier) + elif isinstance(event, PubREL): resp = PubCOMP(packet_identifier=event.packet_identifier) self._send_packet(resp) From 9c8239a7531e0b1df346d49c06338e207536de59 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 22 Oct 2016 17:58:43 +1100 Subject: [PATCH 12/48] to client QoS 2, now to just patch up the branches --- crossbar/_log_categories.py | 9 +++++ crossbar/adapter/mqtt/protocol.py | 1 + crossbar/adapter/mqtt/tx.py | 62 +++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/crossbar/_log_categories.py b/crossbar/_log_categories.py index 667a95cb2..3a8d04b3e 100644 --- a/crossbar/_log_categories.py +++ b/crossbar/_log_categories.py @@ -74,6 +74,9 @@ "MQ202": "Received a QoS 1 Publish from '{client_id}'", "MQ203": "Received a QoS 2 Publish from '{client_id}'", "MQ300": "Got a PubACK for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", + "MQ301": "Got a PubREC for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", + "MQ302": "Got a PubREC for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", + "MQ302": "Got a PubCOMP for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", "MQ400": "MQTT client '{client_id}' timed out after recieving no full packets for {seconds}", "MQ401": "Protocol violation from '{client_id}', terminating connection: {error}", "MQ402": "Got a packet ('{packet_id}') from '{client_id}' that is invalid for a server, terminating connection", @@ -84,6 +87,12 @@ "MQ503": "Error handling a QoS 0 Publish from '{client_id}', dropping connection", "MQ504": "Error handling a QoS 1 Publish from '{client_id}', dropping connection", "MQ505": "Error handling a QoS 2 Publish from '{client_id}', dropping connection", + "MQ506": "Client '{client_id}' sent a PubACK for a non-QoS 1 Publish, dropping connection", + "MQ507": "Client '{client_id}' sent a PubREC for a non-QoS 2 Publish, dropping connection", + "MQ508": "Client '{client_id}' sent a duplicate PubREC, dropping connection", + "MQ509": "Client '{client_id}' sent a PubCOMP for a non-QoS 2 Publish, dropping connection", + "MQ510": "Client '{client_id}' sent a PubCOMP before a PubREC, dropping connection", + } diff --git a/crossbar/adapter/mqtt/protocol.py b/crossbar/adapter/mqtt/protocol.py index 7825561d1..41a1af4cf 100644 --- a/crossbar/adapter/mqtt/protocol.py +++ b/crossbar/adapter/mqtt/protocol.py @@ -91,6 +91,7 @@ class _NeedMoreData(Exception): P_UNSUBACK: UnsubACK, P_PINGRESP: PingRESP, P_PUBREC: PubREC, + P_PUBREL: PubREL, P_PUBCOMP: PubCOMP, } diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index b71176dbb..666923311 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -430,6 +430,13 @@ def _handle_events(self, events): elif isinstance(event, PubACK): if event.packet_identifier in self.session._publishes_awaiting_ack: + + if not self.session._publishes_awaiting_ack[event.packet_identifier].qos == 1: + self.log.error(log_category="MQ506", + client_id=self.session.client_id) + self.transport.loseConnection() + returnValue(None) + # MQTT-4.3.2-1: Only acknowledge when it has been PubACK'd del self.session._publishes_awaiting_ack[event.packet_identifier] self.session._in_flight_packet_ids.remove(event.packet_identifier) @@ -439,11 +446,66 @@ def _handle_events(self, events): log_category="MQ300", client_id=self.session.client_id, pub_id=event.packet_identifier) + elif isinstance(event, PubREC): + + if event.packet_identifier in self.session._publishes_awaiting_ack: + + if not self.session._publishes_awaiting_ack[event.packet_identifier].qos == 2: + self.log.error(log_category="MQ507", + client_id=self.session.client_id) + self.transport.loseConnection() + returnValue(None) + + if not self.session._publishes_awaiting_ack[event.packet_identifier].stage == 0: + self.log.error(log_category="MQ508", + client_id=self.session.client_id) + self.transport.loseConnection() + returnValue(None) + + # MQTT-4.3.3-1: Send back a PubREL + self.session._publishes_awaiting_ack[event.packet_identifier].stage = 1 + + resp = PubREL(packet_identifier=event.packet_identifier) + self._send_packet(resp) + + else: + self.log.warn( + log_category="MQ301", client_id=self.session.client_id, + pub_id=event.packet_identifier) + elif isinstance(event, PubREL): resp = PubCOMP(packet_identifier=event.packet_identifier) self._send_packet(resp) continue + elif isinstance(event, PubCOMP): + + if event.packet_identifier in self.session._publishes_awaiting_ack: + + if not self.session._publishes_awaiting_ack[event.packet_identifier].qos == 2: + self.log.error(log_category="MQ509", + client_id=self.session.client_id) + self.transport.loseConnection() + returnValue(None) + + if not self.session._publishes_awaiting_ack[event.packet_identifier].stage == 1: + self.log.error(log_category="MQ510", + client_id=self.session.client_id) + self.transport.loseConnection() + returnValue(None) + + # MQTT-4.3.3-1: Send back a PubCOMP, release the packet + del self.session._publishes_awaiting_ack[event.packet_identifier] + self.session._in_flight_packet_ids.remove(event.packet_identifier) + + resp = PubCOMP(packet_identifier=event.packet_identifier) + self._send_packet(resp) + + else: + self.log.warn( + log_category="MQ302", client_id=self.session.client_id, + pub_id=event.packet_identifier) + else: if isinstance(event, Failure): self.log.error( From 7fd2b971e349040cedf303c31b230ffa64df988e Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sun, 23 Oct 2016 16:12:56 +1100 Subject: [PATCH 13/48] update requirements --- Makefile | 44 ++------------------------------------------ requirements-in.txt | 4 ++-- 2 files changed, 4 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index bb6c867c6..f29de9ff0 100644 --- a/Makefile +++ b/Makefile @@ -42,48 +42,8 @@ freeze: clean pip install --no-cache-dir -r requirements-in.txt pip freeze -r requirements-in.txt pip install hashin - cat requirements-in.txt | grep -v crossbar | grep -v hashin > requirements.txt - # FIXME: hashin each dependency in requirements.txt and remove the original entries (so no double entries are left) - hashin click - hashin setuptools - hashin zope.interface - hashin Twisted - hashin autobahn - hashin netaddr - hashin PyTrie - hashin Jinja2 - hashin mistune - hashin Pygments - hashin PyYAML - hashin shutilwhich - hashin sdnotify - hashin psutil - hashin lmdb - hashin u-msgpack-python - hashin cbor - hashin py-ubjson - hashin cryptography - hashin pyOpenSSL - hashin pyasn1 - hashin pyasn1-modules - hashin service-identity - hashin PyNaCl - hashin treq - hashin setproctitle - hashin pyqrcode - hashin watchdog - hashin argh - hashin attrs - hashin cffi - hashin enum34 - hashin idna - hashin ipaddress - hashin MarkupSafe - hashin pathtools - hashin pycparser - hashin requests - hashin six - hashin txaio + rm requirements.txt + cat requirements-in.txt | sed -e "s/>=/==/g" | xargs hashin -v > requirements.txt wheel: LMDB_FORCE_CFFI=1 SODIUM_INSTALL=bundled pip wheel --require-hashes --wheel-dir ./wheels -r requirements.txt diff --git a/requirements-in.txt b/requirements-in.txt index e7ad23055..da6e047d4 100644 --- a/requirements-in.txt +++ b/requirements-in.txt @@ -1,7 +1,7 @@ click>=6.6 setuptools>=28.3.0 zope.interface>=4.3.2 -Twisted>=16.5.0rc1 +Twisted>=16.5.0rc2 autobahn>=0.16.0 netaddr>=0.7.18 PyTrie>=0.2 @@ -28,5 +28,5 @@ pyqrcode>=1.2.1 watchdog>=0.8.3 bitstring>=3.1.5 attrs>=16.2.0 -incremental>=16.10.0 +incremental>=16.10.1 constantly>=15.1.0 From 3ef3499fc8fb5fd05d43ddce4dfab44e6c5cd672 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sun, 23 Oct 2016 16:40:29 +1100 Subject: [PATCH 14/48] make some stuff parameterised --- crossbar/adapter/mqtt/test/test_tx.py | 322 +++++--------------------- 1 file changed, 64 insertions(+), 258 deletions(-) diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index 78421589a..cf250bbba 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -35,7 +35,8 @@ from functools import partial from binascii import unhexlify -from crossbar.adapter.mqtt.tx import MQTTServerTwistedProtocol, AwaitingACK +from crossbar.adapter.mqtt.tx import ( + MQTTServerTwistedProtocol, AwaitingACK, Session) from crossbar.adapter.mqtt.protocol import ( MQTTParser, client_packet_handlers, P_CONNACK) from crossbar.adapter.mqtt._events import ( @@ -74,9 +75,23 @@ def existing_wamp_session(self, event): return None +def make_test_items(handler, sessions=None): + + sessions = sessions or {} + + r = Clock() + t = StringTransport() + p = MQTTServerTwistedProtocol(handler, r, sessions) + cp = MQTTClientParser() + + p.makeConnection(t) + + return sessions, r, t, p, cp + + class TwistedProtocolLoggingTests(TestCase): """ - Tests for the logging functionality of the Twisted MQTT protocol.b + Tests for the logging functionality of the Twisted MQTT protocol. """ def test_send_packet(self): @@ -84,15 +99,8 @@ def test_send_packet(self): On sending a packet, a trace log message is emitted with details of the sent packet. """ - sessions = {} - h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( # CONNECT @@ -118,14 +126,8 @@ def test_recv_packet(self): On receiving a packet, a trace log message is emitted with details of the received packet. """ - sessions = {} - h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( # CONNECT @@ -152,14 +154,8 @@ def test_keepalive(self): Compliance statement MQTT-3.1.2-24 """ - sessions = {} - h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( # CONNECT, with keepalive of 2 @@ -187,14 +183,8 @@ def test_keepalive_canceled_on_lost_connection(self): If a client connects with a timeout, and disconnects themselves, we will remove the timeout. """ - sessions = {} - h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( # CONNECT, with keepalive of 2 @@ -223,14 +213,8 @@ def test_keepalive_requires_full_packet(self): Compliance statement MQTT-3.1.2-24 """ - sessions = {} - h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( # CONNECT, with keepalive of 2 @@ -272,14 +256,8 @@ def test_keepalive_full_packet_resets_timeout(self): keep_alive * 1.5, the connection will remain, and the timeout will be reset. """ - sessions = {} - h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( # CONNECT, with keepalive of 2 @@ -320,15 +298,8 @@ def test_only_unique(self): Compliance statement MQTT-3.1.3-2 """ - sessions = {} - h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( # CONNECT, client ID of test123 @@ -349,12 +320,7 @@ def test_only_unique(self): }) # New session - r2 = Clock() - t2 = StringTransport() - p2 = MQTTServerTwistedProtocol(h, r2, sessions) - cp2 = MQTTClientParser() - - p2.makeConnection(t2) + sessions, r2, t2, p2, cp2 = make_test_items(h, sessions=sessions) # Send the same connect, with the same client ID for x in iterbytes(data): @@ -374,15 +340,8 @@ def test_allow_connects_with_same_id_if_disconnected(self): If a client connects and there is an existing session which is disconnected, it may connect. """ - sessions = {} - h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -405,12 +364,7 @@ def test_allow_connects_with_same_id_if_disconnected(self): p.connectionLost(None) # New session - r2 = Clock() - t2 = StringTransport() - p2 = MQTTServerTwistedProtocol(h, r2, sessions) - cp2 = MQTTClientParser() - - p2.makeConnection(t2) + sessions, r2, t2, p2, cp2 = make_test_items(h, sessions=sessions) # Send the same connect, with the same client ID for x in iterbytes(data): @@ -436,14 +390,8 @@ def test_clean_session_destroys_session(self): Compliance statement MQTT-3.2.2-1 """ - sessions = {} - h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -466,12 +414,7 @@ def test_clean_session_destroys_session(self): flags=ConnectFlags(clean_session=True)).serialise() ) - r2 = Clock() - t2 = StringTransport() - p2 = MQTTServerTwistedProtocol(h, r2, sessions) - cp2 = MQTTClientParser() - - p2.makeConnection(t2) + sessions, r2, t2, p2, cp2 = make_test_items(h, sessions=sessions) # Send the same connect, with the same client ID for x in iterbytes(data): @@ -503,17 +446,10 @@ def test_transport_paused_while_processing(self): The transport is paused whilst the MQTT protocol is parsing/handling existing items. """ - sessions = {} - d = Deferred() h = BasicHandler() h.process_connect = lambda x: d - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - - t.connected = True - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -536,16 +472,9 @@ def test_unknown_connect_code_must_lose_connection(self): Compliance statements MQTT-3.2.2-4, MQTT-3.2.2-5 """ - sessions = {} - d = Deferred() h = BasicHandler(6) - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -565,14 +494,8 @@ def test_lose_conn_on_protocol_violation(self): Compliance statement MQTT-4.8.0-1 """ - sessions = {} - h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( # Invalid CONNECT @@ -599,8 +522,6 @@ def test_lose_conn_on_unimplemented_packet(self): Compliance statement: MQTT-4.8.0-1 """ - sessions = {} - # This shouldn't normally happen, but just in case. from crossbar.adapter.mqtt import protocol protocol.server_packet_handlers[protocol.P_SUBACK] = SubACK @@ -608,11 +529,7 @@ def test_lose_conn_on_unimplemented_packet(self): lambda: protocol.server_packet_handlers.pop(protocol.P_SUBACK)) h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -638,14 +555,8 @@ def test_lose_conn_on_reserved_qos3(self): Compliance statement: MQTT-3.3.1-4 """ - sessions = {} - h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) conn = Connect(client_id=u"test123", flags=ConnectFlags(clean_session=False)) @@ -662,6 +573,18 @@ def test_lose_conn_on_reserved_qos3(self): self.assertTrue(t.disconnecting) + def test_packet_id_is_sixteen_bit(self): + """ + The packet ID generator makes IDs that fit within a 16bit uint. + """ + session = Session(client_id=u"test123", wamp_session=None) + session_id = session.get_packet_id() + self.assertTrue(session_id > -1) + self.assertTrue(session_id < 65536) + + # And it is a valid session ID... + SubACK(session_id, [1]).serialise() + class NonZeroConnACKTests(object): @@ -674,16 +597,9 @@ def test_non_zero_connect_code_must_have_no_present_session(self): Compliance statement MQTT-3.2.2-4 """ - sessions = {} - d = Deferred() h = BasicHandler(self.connect_code) - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -728,19 +644,12 @@ def test_subscribe_always_gets_packet(self): Compliance statements MQTT-3.8.4-1 """ - sessions = {} - class SubHandler(BasicHandler): def process_subscribe(self, event): return succeed([128]) h = SubHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -765,19 +674,12 @@ def test_subscribe_same_id(self): Compliance statements MQTT-3.8.4-2 """ - sessions = {} - class SubHandler(BasicHandler): def process_subscribe(self, event): return succeed([0]) h = SubHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -803,20 +705,13 @@ def test_exception_in_subscribe_drops_connection(self): Compliance statement MQTT-4.8.0-2 """ - sessions = {} - class SubHandler(BasicHandler): @inlineCallbacks def process_subscribe(self, event): raise Exception("boom!") h = SubHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -850,7 +745,6 @@ def test_got_sent_packet(self): """ `process_connect` on the handler will get the correct Connect packet. """ - sessions = {} got_packets = [] class SubHandler(BasicHandler): @@ -859,12 +753,7 @@ def process_connect(self_, event): return succeed(0) h = SubHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -885,20 +774,13 @@ def test_exception_in_connect_drops_connection(self): Compliance statement MQTT-4.8.0-2 """ - sessions = {} - class SubHandler(BasicHandler): @inlineCallbacks def process_connect(self, event): raise Exception("boom!") h = SubHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -932,19 +814,12 @@ def test_exception_in_connect_drops_connection(self): Compliance statement MQTT-4.8.0-2 """ - sessions = {} - class SubHandler(BasicHandler): def process_unsubscribe(self, event): raise Exception("boom!") h = SubHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -976,7 +851,6 @@ def test_unsubscription_gets_unsuback_with_same_id(self): Compliance statements MQTT-3.10.4-4, MQTT-3.10.4-5, MQTT-3.12.4-1 """ - sessions = {} got_packets = [] class SubHandler(BasicHandler): @@ -985,12 +859,7 @@ def process_unsubscribe(self, event): return succeed(None) h = SubHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) unsub = Unsubscribe(packet_identifier=1234, topics=[u"foo"]).serialise() @@ -1022,7 +891,6 @@ def test_qos_0_sends_no_ack(self): """ When a QoS 0 Publish packet is recieved, we don't send back a PubACK. """ - sessions = {} got_packets = [] class PubHandler(BasicHandler): @@ -1031,12 +899,7 @@ def process_publish_qos_0(self, event): return succeed(None) h = PubHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) pub = Publish(duplicate=False, qos_level=0, retain=False, topic_name=u"foo", packet_identifier=None, @@ -1074,19 +937,12 @@ def test_qos_0_failure_drops_connection(self): Compliance statement MQTT-4.8.0-2 """ - sessions = {} - class PubHandler(BasicHandler): def process_publish_qos_0(self, event): raise Exception("boom!") h = PubHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -1121,7 +977,6 @@ def test_qos_1_sends_ack(self): Compliance statement MQTT-3.3.4-1 Spec part 3.4 """ - sessions = {} got_packets = [] class PubHandler(BasicHandler): @@ -1130,12 +985,7 @@ def process_publish_qos_1(self, event): return succeed(None) h = PubHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) pub = Publish(duplicate=False, qos_level=1, retain=False, topic_name=u"foo", packet_identifier=2345, @@ -1174,19 +1024,12 @@ def test_qos_1_failure_drops_connection(self): Compliance statement MQTT-4.8.0-2 """ - sessions = {} - class PubHandler(BasicHandler): def process_publish_qos_1(self, event): raise Exception("boom!") h = PubHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -1222,7 +1065,6 @@ def test_qos_2_sends_ack(self): Compliance statement MQTT-4.3.3-2 Spec part 3.4, 4.3.3 """ - sessions = {} got_packets = [] class PubHandler(BasicHandler): @@ -1231,12 +1073,7 @@ def process_publish_qos_2(self, event): return succeed(None) h = PubHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) pub = Publish(duplicate=False, qos_level=2, retain=False, topic_name=u"foo", packet_identifier=2345, @@ -1290,19 +1127,12 @@ def test_qos_2_failure_drops_connection(self): Compliance statement MQTT-4.8.0-2 """ - sessions = {} - class PubHandler(BasicHandler): def process_publish_qos_2(self, event): raise Exception("boom!") h = PubHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -1340,16 +1170,10 @@ def test_qos_0_queues_message(self): The WAMP layer calling send_publish will queue a message up for sending, and send it next time it has a chance. """ - sessions = {} got_packets = [] h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -1392,16 +1216,10 @@ def test_qos_1_queues_message(self): The WAMP layer calling send_publish will queue a message up for sending, and send it next time it has a chance. """ - sessions = {} got_packets = [] h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -1469,16 +1287,10 @@ def test_qos_1_unassociated_puback_warns(self): XXX: Spec is unclear if this is the proper behaviour! """ - sessions = {} got_packets = [] h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", @@ -1516,16 +1328,10 @@ def test_qos_2_queues_message(self): The WAMP layer calling send_publish will queue a message up for sending, and send it next time it has a chance. """ - sessions = {} got_packets = [] h = BasicHandler() - r = Clock() - t = StringTransport() - p = MQTTServerTwistedProtocol(h, r, sessions) - cp = MQTTClientParser() - - p.makeConnection(t) + sessions, r, t, p, cp = make_test_items(h) data = ( Connect(client_id=u"test123", From d8b8c1bd6d93fdc3ca156bc855fa8ec06b8709ab Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sun, 23 Oct 2016 17:41:34 +1100 Subject: [PATCH 15/48] qos1 resending test --- crossbar/adapter/mqtt/test/test_tx.py | 87 ++++++++++++++++++++++++++- crossbar/adapter/mqtt/tx.py | 2 +- 2 files changed, 86 insertions(+), 3 deletions(-) diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index cf250bbba..0a4f5fdd9 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -1328,8 +1328,6 @@ def test_qos_2_queues_message(self): The WAMP layer calling send_publish will queue a message up for sending, and send it next time it has a chance. """ - got_packets = [] - h = BasicHandler() sessions, r, t, p, cp = make_test_items(h) @@ -1405,3 +1403,88 @@ def test_qos_2_queues_message(self): self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 0) self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) self.assertFalse(t.disconnecting) + + def test_qos_1_resent_on_disconnect(self): + """ + If we send a QoS1 Publish and we did not get a PubACK from the client + before it disconnected, we will resend the Publish packet if it + connects with a non-clean session. + + Compliance statements: MQTT-4.4.0-1, MQTT-3.3.1-1 + """ + h = BasicHandler() + sessions, r, t, p, cp = make_test_items(h) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + ) + + for x in iterbytes(data): + p.dataReceived(x) + + # No queued messages + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + + # Make the packet ID be deterministic + sessions[u"test123"].get_packet_id = lambda: 4567 + + # WAMP layer calls send_publish, with QoS 1 + p.send_publish(u"hello", 1, b'some bytes') + + # Nothing should have been sent yet, it is queued + self.assertEqual(len(sessions[u"test123"].queued_messages), 1) + + # Advance the clock + r.advance(0.1) + + # We should now get the sent Publish + expected_publish = Publish(duplicate=False, qos_level=1, retain=False, + packet_identifier=4567, topic_name=u"hello", + payload=b"some bytes") + events = cp.data_received(t.value()) + t.clear() + self.assertEqual(len(events), 2) + self.assertEqual(events[1], expected_publish) + + # Server is still awaiting the client's response, message is not queued + self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) + self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], + AwaitingACK(qos=1, stage=0, message=expected_publish)) + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + + # Disconnect the client + t.connected = False + t.loseConnection() + p.connectionLost(None) + + sessions, r2, t2, p2, cp2 = make_test_items(h, sessions=sessions) + + # We must NOT have a clean session + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=False)).serialise() + ) + + for x in iterbytes(data): + p2.dataReceived(x) + + # We should have two events; the ConnACK, and the Publish. The ConnACK + # MUST come first. + events = cp2.data_received(t2.value()) + self.assertEqual(len(events), 2) + self.assertIsInstance(events[0], ConnACK) + self.assertIsInstance(events[1], Publish) + + # The Publish packet must have DUP set to True. + resent_publish = Publish(duplicate=True, qos_level=2, retain=False, + packet_identifier=4567, topic_name=u"hello", + payload=b"some bytes") + self.assertEqual(events[1], resent_publish) + + # It is no longer queued + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 0) + self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + self.assertFalse(t.disconnecting) diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index 666923311..7d16bec8a 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -66,7 +66,7 @@ class Session(object): connected = attr.ib(default=False) survives = attr.ib(default=False) _in_flight_packet_ids = attr.ib(default=attr.Factory(set)) - _publishes_awaiting_ack = attr.ib(default=attr.Factory(dict)) + _publishes_awaiting_ack = attr.ib(default=attr.Factory(collections.OrderedDict)) def get_packet_id(self): x = 0 From 0f99b1004f29cdb2952079fe0ca73615bb914b35 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sun, 23 Oct 2016 17:51:03 +1100 Subject: [PATCH 16/48] resend logic for QoS 1 works --- crossbar/adapter/mqtt/test/test_tx.py | 21 ++++++++++++++++++--- crossbar/adapter/mqtt/tx.py | 14 ++++++++++---- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index 0a4f5fdd9..b34dfa57b 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -1417,7 +1417,7 @@ def test_qos_1_resent_on_disconnect(self): data = ( Connect(client_id=u"test123", - flags=ConnectFlags(clean_session=True)).serialise() + flags=ConnectFlags(clean_session=False)).serialise() ) for x in iterbytes(data): @@ -1470,21 +1470,36 @@ def test_qos_1_resent_on_disconnect(self): for x in iterbytes(data): p2.dataReceived(x) + # The flushing is queued, so we'll have to spin the reactor + self.assertEqual(p2._flush_publishes.args, (True,)) + + r2.advance(0.1) + # We should have two events; the ConnACK, and the Publish. The ConnACK # MUST come first. events = cp2.data_received(t2.value()) + t2.clear() self.assertEqual(len(events), 2) self.assertIsInstance(events[0], ConnACK) self.assertIsInstance(events[1], Publish) # The Publish packet must have DUP set to True. - resent_publish = Publish(duplicate=True, qos_level=2, retain=False, + resent_publish = Publish(duplicate=True, qos_level=1, retain=False, packet_identifier=4567, topic_name=u"hello", payload=b"some bytes") self.assertEqual(events[1], resent_publish) + # We send the PubACK to this Publish + puback = PubACK(packet_identifier=4567) + + for x in iterbytes(puback.serialise()): + p2.dataReceived(x) + + events = cp2.data_received(t2.value()) + self.assertEqual(len(events), 0) + # It is no longer queued self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 0) self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) self.assertEqual(len(sessions[u"test123"].queued_messages), 0) - self.assertFalse(t.disconnecting) + self.assertFalse(t2.disconnecting) diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index 7d16bec8a..f043b81fd 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -99,8 +99,6 @@ def __init__(self, handler, reactor, mqtt_sessions, _id_maker=_ids): self._reactor = reactor self._mqtt = MQTTParser() self._handler = handler - self._in_flight_publishes = set() - self._waiting_for_ack = {} self._timeout = None self._timeout_time = 0 self._mqtt_sessions = mqtt_sessions @@ -194,7 +192,7 @@ def _send_packet(self, packet): packet=packet, conn_id=self._connection_id) self.transport.write(packet.serialise()) - def _flush_saved_messages(self): + def _flush_saved_messages(self, including_non_acked=False): if self._flush_publishes: self._flush_publishes = None @@ -203,6 +201,13 @@ def _flush_saved_messages(self): if not self.transport.connected: return None + if including_non_acked: + for message in self.session._publishes_awaiting_ack.values(): + if message.qos == 1: + message.message.duplicate = True + self._send_packet(message.message) + + # New, queued messages while self.session.queued_messages: message = self.session.queued_messages.popleft() self._send_publish(message.topic, message.qos, message.body) @@ -274,7 +279,8 @@ def _handle_events(self, events): if event.client_id in self._mqtt_sessions: self.session = self._mqtt_sessions[event.client_id] - self._reactor.callLater(0, self._flush_saved_messages) + self._flush_publishes = self._reactor.callLater( + 0, self._flush_saved_messages, True) self._handler.existing_wamp_session(self.session) # Have a session, set to 1/True as in MQTT-3.2.2-2 session_present = True From efee101196848f910dd04c20f48daab318a39812 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sun, 23 Oct 2016 20:39:56 +1100 Subject: [PATCH 17/48] qos 2 reconnect before pubrel sent --- crossbar/adapter/mqtt/test/test_tx.py | 129 ++++++++++++++++++++++++++ crossbar/adapter/mqtt/tx.py | 20 +++- 2 files changed, 144 insertions(+), 5 deletions(-) diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index b34dfa57b..0ee0fff04 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -1503,3 +1503,132 @@ def test_qos_1_resent_on_disconnect(self): self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) self.assertEqual(len(sessions[u"test123"].queued_messages), 0) self.assertFalse(t2.disconnecting) + + def test_qos_2_resent_on_disconnect_pubrel(self): + """ + If we send a QoS2 Publish and we did not get a PubREL from the client + before it disconnected, we will resend the Publish packet if it + connects with a non-clean session. + + Compliance statements: MQTT-4.4.0-1, MQTT-3.3.1-1 + """ + h = BasicHandler() + sessions, r, t, p, cp = make_test_items(h) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=False)).serialise() + ) + + for x in iterbytes(data): + p.dataReceived(x) + + # No queued messages + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + + # Make the packet ID be deterministic + sessions[u"test123"].get_packet_id = lambda: 4567 + + # WAMP layer calls send_publish, with QoS 2 + p.send_publish(u"hello", 2, b'some bytes') + + # Nothing should have been sent yet, it is queued + self.assertEqual(len(sessions[u"test123"].queued_messages), 1) + + # Advance the clock + r.advance(0.1) + + # We should now get the sent Publish + expected_publish = Publish(duplicate=False, qos_level=2, retain=False, + packet_identifier=4567, topic_name=u"hello", + payload=b"some bytes") + events = cp.data_received(t.value()) + t.clear() + self.assertEqual(len(events), 2) + self.assertEqual(events[1], expected_publish) + + # Server is still awaiting the client's response, message is not queued + self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) + self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], + AwaitingACK(qos=2, stage=0, message=expected_publish)) + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + + # Disconnect the client + t.connected = False + t.loseConnection() + p.connectionLost(None) + + sessions, r2, t2, p2, cp2 = make_test_items(h, sessions=sessions) + + # We must NOT have a clean session + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=False)).serialise() + ) + + for x in iterbytes(data): + p2.dataReceived(x) + + # The flushing is queued, so we'll have to spin the reactor + self.assertEqual(p2._flush_publishes.args, (True,)) + + r2.advance(0.1) + + # We should have two events; the ConnACK, and the Publish. The ConnACK + # MUST come first. + events = cp2.data_received(t2.value()) + t2.clear() + self.assertEqual(len(events), 2) + self.assertIsInstance(events[0], ConnACK) + self.assertIsInstance(events[1], Publish) + + # The Publish packet must have DUP set to True. + resent_publish = Publish(duplicate=True, qos_level=2, retain=False, + packet_identifier=4567, topic_name=u"hello", + payload=b"some bytes") + self.assertEqual(events[1], resent_publish) + + # Server is still waiting for us to send the PubREC -- QoS2, Stage 0 + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) + self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + self.assertFalse(t2.disconnecting) + self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], + AwaitingACK(qos=2, stage=0, message=resent_publish)) + + # We send the PubREC to this Publish + pubrec = PubREC(packet_identifier=4567) + + for x in iterbytes(pubrec.serialise()): + p2.dataReceived(x) + + # Should get a PubREL back + events = cp2.data_received(t2.value()) + t2.clear() + self.assertEqual(len(events), 1) + self.assertEqual(events[0], PubREL(packet_identifier=4567)) + + # Server is still waiting for us to send the PubCOMP -- QoS2, Stage 1 + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) + self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + self.assertFalse(t2.disconnecting) + self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], + AwaitingACK(qos=2, stage=1, message=resent_publish)) + + # We send the PubCOMP to this Publish + pubcomp = PubCOMP(packet_identifier=4567) + + for x in iterbytes(pubcomp.serialise()): + p2.dataReceived(x) + + # No more packets sent to us + events = cp2.data_received(t2.value()) + self.assertEqual(len(events), 0) + + # Not queued, not awaiting ACK + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 0) + self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + self.assertFalse(t2.disconnecting) diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index f043b81fd..703d794b4 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -206,6 +206,19 @@ def _flush_saved_messages(self, including_non_acked=False): if message.qos == 1: message.message.duplicate = True self._send_packet(message.message) + if message.qos == 2: + # Stage 0 == Publish sent + if message.stage == 0: + message.message.duplicate = True + self._send_packet(message.message) + + # Stage 1 == PubREC got, PubREL sent + elif message.stage == 1: + pass + + # Invalid! + else: + pass # New, queued messages while self.session.queued_messages: @@ -443,7 +456,7 @@ def _handle_events(self, events): self.transport.loseConnection() returnValue(None) - # MQTT-4.3.2-1: Only acknowledge when it has been PubACK'd + # MQTT-4.3.2-1: Release the packet ID del self.session._publishes_awaiting_ack[event.packet_identifier] self.session._in_flight_packet_ids.remove(event.packet_identifier) @@ -500,13 +513,10 @@ def _handle_events(self, events): self.transport.loseConnection() returnValue(None) - # MQTT-4.3.3-1: Send back a PubCOMP, release the packet + # MQTT-4.3.3-1: Release the packet ID del self.session._publishes_awaiting_ack[event.packet_identifier] self.session._in_flight_packet_ids.remove(event.packet_identifier) - resp = PubCOMP(packet_identifier=event.packet_identifier) - self._send_packet(resp) - else: self.log.warn( log_category="MQ302", client_id=self.session.client_id, From 087b792e87f5a59abbee856a97143e487fecdb6c Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sun, 23 Oct 2016 21:22:14 +1100 Subject: [PATCH 18/48] pubcomp resend --- crossbar/adapter/mqtt/test/test_tx.py | 122 ++++++++++++++++++++++++++ crossbar/adapter/mqtt/tx.py | 9 +- 2 files changed, 128 insertions(+), 3 deletions(-) diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index 0ee0fff04..6261ad0c1 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -1632,3 +1632,125 @@ def test_qos_2_resent_on_disconnect_pubrel(self): self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) self.assertEqual(len(sessions[u"test123"].queued_messages), 0) self.assertFalse(t2.disconnecting) + + def test_qos_2_resent_on_disconnect_pubcomp(self): + """ + If we send a QoS2 Publish and we did not get a PubCOMP from the client + before it disconnected, we will resend the PubREL packet if it + connects with a non-clean session. + + Compliance statements: MQTT-4.4.0-1, MQTT-3.3.1-1 + """ + h = BasicHandler() + sessions, r, t, p, cp = make_test_items(h) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=False)).serialise() + ) + + for x in iterbytes(data): + p.dataReceived(x) + + # No queued messages + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + + # Make the packet ID be deterministic + sessions[u"test123"].get_packet_id = lambda: 4567 + + # WAMP layer calls send_publish, with QoS 2 + p.send_publish(u"hello", 2, b'some bytes') + + # Nothing should have been sent yet, it is queued + self.assertEqual(len(sessions[u"test123"].queued_messages), 1) + + # Advance the clock + r.advance(0.1) + + # We should now get the sent Publish + expected_publish = Publish(duplicate=False, qos_level=2, retain=False, + packet_identifier=4567, topic_name=u"hello", + payload=b"some bytes") + events = cp.data_received(t.value()) + t.clear() + self.assertEqual(len(events), 2) + self.assertEqual(events[1], expected_publish) + + # Server is waiting for us to send the PubREC -- QoS2, Stage 0 + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) + self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + self.assertFalse(t.disconnecting) + self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], + AwaitingACK(qos=2, stage=0, message=expected_publish)) + + # We send the PubREC to this Publish + pubrec = PubREC(packet_identifier=4567) + + for x in iterbytes(pubrec.serialise()): + p.dataReceived(x) + + # Should get a PubREL back + events = cp.data_received(t.value()) + t.clear() + self.assertEqual(len(events), 1) + self.assertEqual(events[0], PubREL(packet_identifier=4567)) + + # Server is waiting for a PubCOMP -- QoS2, Stage 1 + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) + self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + self.assertFalse(t.disconnecting) + self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], + AwaitingACK(qos=2, stage=1, message=expected_publish)) + + # Disconnect the client + t.connected = False + t.loseConnection() + p.connectionLost(None) + + sessions, r2, t2, p2, cp2 = make_test_items(h, sessions=sessions) + + # We must NOT have a clean session + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=False)).serialise() + ) + + for x in iterbytes(data): + p2.dataReceived(x) + + # The flushing is queued, so we'll have to spin the reactor + self.assertEqual(p2._flush_publishes.args, (True,)) + r2.advance(0.1) + + # Should get a resent PubREL back + events = cp2.data_received(t2.value()) + t2.clear() + self.assertEqual(len(events), 2) + self.assertIsInstance(events[0], ConnACK) + self.assertEqual(events[1], PubREL(packet_identifier=4567)) + + # Server is still waiting for us to send the PubCOMP -- QoS2, Stage 1 + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) + self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + self.assertFalse(t2.disconnecting) + self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], + AwaitingACK(qos=2, stage=1, message=expected_publish)) + + # We send the PubCOMP to this PubREL + pubcomp = PubCOMP(packet_identifier=4567) + + for x in iterbytes(pubcomp.serialise()): + p2.dataReceived(x) + + # No more packets sent to us + events = cp2.data_received(t2.value()) + self.assertEqual(len(events), 0) + + # Not queued, not awaiting ACK + self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 0) + self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + self.assertFalse(t2.disconnecting) diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index 703d794b4..788286a24 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -207,14 +207,17 @@ def _flush_saved_messages(self, including_non_acked=False): message.message.duplicate = True self._send_packet(message.message) if message.qos == 2: - # Stage 0 == Publish sent if message.stage == 0: + # Stage 0 == Publish sent + # Resend Publish message.message.duplicate = True self._send_packet(message.message) - # Stage 1 == PubREC got, PubREL sent elif message.stage == 1: - pass + # Stage 1 == PubREC got, PubREL sent + # Resend PubREL + pkt = PubREL(packet_identifier=message.message.packet_identifier) + self._send_packet(pkt) # Invalid! else: From 5d45b86f9b94d605455989d21a05eb6e095bbcc5 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Mon, 24 Oct 2016 16:13:44 +1100 Subject: [PATCH 19/48] stop things importing the default reactor (which makes CB actually load KQueue on BSD/macOS now) --- crossbar/adapter/rest/callee.py | 9 ++++++--- crossbar/adapter/rest/subscriber.py | 8 ++++++-- crossbar/common/fswatcher.py | 4 ++-- crossbar/common/process.py | 12 +++++++----- crossbar/common/profiler.py | 2 +- crossbar/twisted/resource.py | 1 - crossbar/worker/router.py | 7 +++++-- 7 files changed, 27 insertions(+), 16 deletions(-) diff --git a/crossbar/adapter/rest/callee.py b/crossbar/adapter/rest/callee.py index 6e1d8d1ab..784adb335 100644 --- a/crossbar/adapter/rest/callee.py +++ b/crossbar/adapter/rest/callee.py @@ -30,8 +30,6 @@ from __future__ import absolute_import -import treq - from six.moves.urllib.parse import urljoin from twisted.internet.defer import inlineCallbacks, returnValue @@ -43,7 +41,12 @@ class RESTCallee(ApplicationSession): def __init__(self, *args, **kwargs): - self._webtransport = kwargs.pop("webTransport", treq) + self._webtransport = kwargs.pop("webTransport") + + if not self._webtransport: + import treq + self._webtransport = treq + super(RESTCallee, self).__init__(*args, **kwargs) @inlineCallbacks diff --git a/crossbar/adapter/rest/subscriber.py b/crossbar/adapter/rest/subscriber.py index 7a8b6cf42..572838a63 100644 --- a/crossbar/adapter/rest/subscriber.py +++ b/crossbar/adapter/rest/subscriber.py @@ -30,7 +30,6 @@ from __future__ import absolute_import -import treq import json from functools import partial @@ -50,7 +49,12 @@ class MessageForwarder(ApplicationSession): log = make_logger() def __init__(self, *args, **kwargs): - self._webtransport = kwargs.pop("webTransport", treq) + self._webtransport = kwargs.pop("webTransport") + + if not self._webtransport: + import treq + self._webtransport = treq + super(MessageForwarder, self).__init__(*args, **kwargs) @inlineCallbacks diff --git a/crossbar/common/fswatcher.py b/crossbar/common/fswatcher.py index 0fc5372ee..a7e2bbbce 100644 --- a/crossbar/common/fswatcher.py +++ b/crossbar/common/fswatcher.py @@ -33,8 +33,6 @@ import os from txaio import make_logger -from twisted.internet import reactor - try: from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler @@ -83,6 +81,8 @@ def on_any_event(evt): u'rel_path': os.path.relpath(evt.src_path, self._working_dir), u'is_directory': evt.is_directory, } + + from twisted.internet import reactor reactor.callFromThread(callback, event) self._handler.on_any_event = on_any_event diff --git a/crossbar/common/process.py b/crossbar/common/process.py index f69faafb4..79e519ff2 100644 --- a/crossbar/common/process.py +++ b/crossbar/common/process.py @@ -44,11 +44,7 @@ # # twisted.conch.manhole_ssh will import even without, but we _need_ SSH import pyasn1 # noqa - from twisted.cred import checkers, portal - from twisted.conch.manhole import ColoredManhole - from twisted.conch.manhole_ssh import ConchFactory, \ - TerminalRealm, \ - TerminalSession + import cryptography # noqa except ImportError as e: _HAS_MANHOLE = False _MANHOLE_MISSING_REASON = str(e) @@ -64,6 +60,8 @@ from txaio import make_logger +from twisted.cred import checkers, portal + from crossbar.common import checkconfig from crossbar.common.checkconfig import get_config_value from crossbar.twisted.endpoint import create_listening_port_from_config @@ -576,6 +574,10 @@ class PatchedTerminalSession(TerminalSession): def windowChanged(self, winSize): pass + from twisted.conch.manhole_ssh import ( + ConchFactory, TerminalRealm, TerminalSession) + from twisted.conch.manhole import ColoredManhole + rlm = TerminalRealm() rlm.sessionFactory = PatchedTerminalSession # monkey patch rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(namespace) diff --git a/crossbar/common/profiler.py b/crossbar/common/profiler.py index af9f2b36c..eb91cd0db 100644 --- a/crossbar/common/profiler.py +++ b/crossbar/common/profiler.py @@ -35,7 +35,6 @@ import six -from twisted.internet import reactor from twisted.internet.defer import Deferred from twisted.internet.threads import deferToThread @@ -221,6 +220,7 @@ def cleanup(res): self.log.info("Starting profiling using {profiler} for {runtime} seconds.", profiler=self._id, runtime=runtime) + from twisted.internet import reactor reactor.callLater(runtime, finish_profile) return self._profile_id, self._finished diff --git a/crossbar/twisted/resource.py b/crossbar/twisted/resource.py index a2fa1e44b..4b67cb31d 100644 --- a/crossbar/twisted/resource.py +++ b/crossbar/twisted/resource.py @@ -37,7 +37,6 @@ from twisted.web.http import NOT_FOUND from twisted.web.resource import Resource, NoResource from twisted.web.static import File -from twisted.web.proxy import ReverseProxyResource # noqa (Republish resource) from twisted.python.filepath import FilePath from txaio import make_logger diff --git a/crossbar/worker/router.py b/crossbar/worker/router.py index 2a9194423..b6ed10a3b 100644 --- a/crossbar/worker/router.py +++ b/crossbar/worker/router.py @@ -90,8 +90,7 @@ from crossbar.twisted.resource import JsonResource, \ Resource404, \ - RedirectResource, \ - ReverseProxyResource + RedirectResource from crossbar.adapter.mqtt.wamp import WampMQTTServerFactory @@ -1069,6 +1068,10 @@ def _create_resource(self, path_config, nested=True): # Reverse proxy resource # elif path_config['type'] == 'reverseproxy': + + # Import late because t.w.proxy imports the reactor + from twisted.web.proxy import ReverseProxyResource + host = path_config['host'] port = int(path_config.get('port', 80)) path = path_config.get('path', '').encode('ascii', 'ignore') From eb54d0905a099572d8c29e037cadf85b6739f101 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Mon, 24 Oct 2016 16:14:11 +1100 Subject: [PATCH 20/48] use plistlib to get the serial number of a macOS machine, fixes #870 --- crossbar/controller/node.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/crossbar/controller/node.py b/crossbar/controller/node.py index a0b3f4331..b2e98c991 100644 --- a/crossbar/controller/node.py +++ b/crossbar/controller/node.py @@ -36,6 +36,8 @@ import getpass import pkg_resources import binascii +import six +import subprocess from collections import OrderedDict import pyqrcode @@ -46,6 +48,7 @@ import twisted from twisted.internet.defer import inlineCallbacks, Deferred from twisted.internet.ssl import optionsForClientTLS +from twisted.python.runtime import platform from txaio import make_logger @@ -162,12 +165,29 @@ def _machine_id(): """ for informational purposes, try to get a machine unique id thing """ - try: - # why this? see: http://0pointer.de/blog/projects/ids.html - with open('/var/lib/dbus/machine-id', 'r') as f: - return f.read().strip() - except: - # OS X? Something else? Get a hostname, at least. + if platform.isLinux(): + try: + # why this? see: http://0pointer.de/blog/projects/ids.html + with open('/var/lib/dbus/machine-id', 'r') as f: + return f.read().strip() + except: + # Non-dbus using Linux, get a hostname + return socket.gethostname() + + elif platform.isMacOSX(): + # Get the serial number of the platform + import plistlib + plist_data = subprocess.check_output(["ioreg", "-rd1", "-c", "IOPlatformExpertDevice", "-a"]) + + if six.PY2: + # Only API on 2.7 + return plistlib.readPlistFromString(plist_data)["IOPlatformSerialNumber"] + else: + # New, non-deprecated 3.4+ API + return plistlib.loads(plist_data)[0]["IOPlatformSerialNumber"] + + else: + # Something else, just get a hostname return socket.gethostname() From 6a2f1b2f2156023a862456d0cbc43a49bf649f34 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Mon, 24 Oct 2016 16:33:57 +1100 Subject: [PATCH 21/48] warn if the loading process would actually do nothing --- crossbar/controller/node.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crossbar/controller/node.py b/crossbar/controller/node.py index b2e98c991..b0a5ccd36 100644 --- a/crossbar/controller/node.py +++ b/crossbar/controller/node.py @@ -414,6 +414,17 @@ def start(self, cdc_mode=False): if not self._config: raise Exception("No node configuration loaded") + if not cdc_mode and not self._config["controller"] and not self._config["workers"]: + self.log.warn( + ("You seem to have no controller config or workers, nor are " + "starting up in CDC mode. Check your config exists, or pass " + "--cdc to `crossbar start`.")) + try: + self._reactor.stop() + except twisted.internet.error.ReactorNotRunning: + pass + return + # get controller config/options # controller_config = self._config.get('controller', {}) From 883dcc2aa765b9b9aff34139f72dff75a1928f47 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sun, 30 Oct 2016 16:49:09 +1100 Subject: [PATCH 22/48] bump min requirements a bit --- requirements-in.txt => requirements-min.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename requirements-in.txt => requirements-min.txt (96%) diff --git a/requirements-in.txt b/requirements-min.txt similarity index 96% rename from requirements-in.txt rename to requirements-min.txt index da6e047d4..4ff400f2b 100644 --- a/requirements-in.txt +++ b/requirements-min.txt @@ -1,7 +1,7 @@ click>=6.6 setuptools>=28.3.0 zope.interface>=4.3.2 -Twisted>=16.5.0rc2 +Twisted>=16.5.0 autobahn>=0.16.0 netaddr>=0.7.18 PyTrie>=0.2 From decb6edb1ebdd19b388ad7df15dfaf81591312ee Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sun, 30 Oct 2016 16:49:49 +1100 Subject: [PATCH 23/48] bump our real requirements --- requirements.txt | 397 +++++++++++++++++++++++------------------------ 1 file changed, 194 insertions(+), 203 deletions(-) diff --git a/requirements.txt b/requirements.txt index 21fb92a6a..349a4bc69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,121 +1,58 @@ -click==6.6 \ - --hash=sha256:cc6a19da8ebff6e7074f731447ef7e112bd23adf3de5c597cf9989f2fd8defe9 -setuptools==28.3.0 \ - --hash=sha256:873b8e61eba51c38266ad8499ec10f669ea9299a2e8d217b086cead83194a495 \ - --hash=sha256:e31c9397fcb3bf20c257f5804d0ea2a17cf220027f4ab65b9ee5158010d41fc2 \ - --hash=sha256:a6425c2dbbc0e3f0741b7e6e18b34896229a538f02f6b1198166b00cd6da19a3 -zope.interface==4.3.2 \ - --hash=sha256:10aba8ab3e011b797656d3a6d0ff8709ee471024f32d6fd056326cca4123e95f \ - --hash=sha256:23eaeb9c59aa41a78cc59b585c80d2eacf96167d575cfd9399f271c967b4b33d \ - --hash=sha256:e727c508a5af11e0bbd690869ed8d3598e6c997e5be66465149f361514a4f3f2 \ - --hash=sha256:ae4963329dc6e78800de72c942422bf641b673fe0708bda7e2a76b96b926121c \ - --hash=sha256:716ee42ea853deb4f6491a0db03e36cf4db82f178ee0f4b3ba433e893d754f16 \ - --hash=sha256:ab357a99d9f00bc43476d70ead78f05d2d7ebc68c8fd8a8a983e2e6e4cb34bfb \ - --hash=sha256:ae3295573d957138585532f2050ca94bed1fdb0fd792f389b1fa6d9cf5df360f \ - --hash=sha256:6d871fb3a7869802fa8ccde0c589e20289e32b30398b24dd46cd1c3bd4dc7382 \ - --hash=sha256:ef296d3f2c26d8b88fd0db57ba13cfcd3df467f333caaeb87f1d31717b14ff87 \ - --hash=sha256:bda6aed260c08c0d1d7c2b100fa3e182e36da064e458af8392acc985fee13644 \ - --hash=sha256:c2fd356ed67d9cd23ed47dddcd118f5b3fba26d4a34eee4bf8974bc80963db98 \ - --hash=sha256:5d78df08d8c571a5ba3c0a95fb9c342487bd63dbe54372224621b556975c5f3e \ - --hash=sha256:7ebc15a8f568162d936da27b8ee01bacf934c3f3758ef5f6a2d4306aeea1337f \ - --hash=sha256:d0e1c5113a10446079f2a8e38e88295b8c34e119d76d30df248bd5b051afdecc \ - --hash=sha256:e270f3a3f3d939ce06e1425e6d70e5d7cc4ccff67e1eaee89c6a6e9d0bdffbfe \ - --hash=sha256:6a0e224a052e3ce27b3a7b1300a24747513f7a507217fcc2a4cb02eb92945cee \ - --hash=sha256:227c4f5908f6ad52b1e1f1748dddb88fb48be32bc4d310eb32032428769dca24 \ - --hash=sha256:cf156d9ec862f1c3a1e56256dc0c3d85048e31f21bc464e03359cba5a1d45250 \ - --hash=sha256:04ae8a1041188faa8de80f9a373e0c39036f4ff5f05b087f45f95e61f5d021e9 \ - --hash=sha256:d5a92d4c10e81d7c2e96a8d6929905b795b46a14fda4b6ec8ec738c02a4f5406 -Twisted==16.5.0rc1 \ - --hash=sha256:3d609d9ea128d1eb84ff4378a2847662103e54cdcd2455921ae20e9968937887 +argh==0.26.2 \ + --hash=sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3 \ + --hash=sha256:e9535b8c84dc9571a48999094fda7f33e63c3f1b74f3e5f3ac0105a58405bb65 +attrs==16.2.0 \ + --hash=sha256:ce9d6cac4705e5aeaca02d3ff72f0006bf9b0a2f29635ae8dab8262e296f6442 \ + --hash=sha256:136f2ec0f94ec77ff2990830feee965d608cab1e8922370e3abdded383d52001 autobahn==0.16.0 \ --hash=sha256:6c9eb70a36ffbc831a1fe76e43f99f5fd26aebfa7c5af2d705c587c4ef20c8f0 \ --hash=sha256:a1dcb4315a0914da56ec484659816de72dfad229be4ac19fa61bbc0111ada884 -netaddr==0.7.18 \ - --hash=sha256:cb305179658334eb035860e515f054504e232b832abb4efc51c04bf8a72d3574 \ - --hash=sha256:98c3d6fe831d119785c37af25f032169dd653a0dad86e54b387f4a5b83da4383 \ - --hash=sha256:a1f5c9fcf75ac2579b9995c843dade33009543c04f218ff7c007b3c81695bd19 \ - --hash=sha256:c64c570ac612e20e8b8a6eee72034c924fff9d76c7a46f50a9f919085f1bfbed -PyTrie==0.2 \ - --hash=sha256:b272021351efadc6757591aac03ed4794bdfd091122204a4673e94bfb66cc500 -Jinja2==2.8 \ - --hash=sha256:1cc03ef32b64be19e0a5b54578dd790906a34943fe9102cfdae0d4495bd536b4 \ - --hash=sha256:bc1ff2ff88dbfacefde4ddde471d1417d3b304e8df103a7a9437d47269201bf4 -mistune==0.7.3 \ - --hash=sha256:ee7447aadcf1962b5af767ff0443dcb0499c16bf73ad36dc99d230e7574571e5 \ - --hash=sha256:21d0e869df3b9189f248e022f1c9763cf9069e1a2f00676f1f1852bd7f98b713 -Pygments==2.1.3 \ - --hash=sha256:485602129949b14247e8b124d28af4654dffbd076537c4a9c44a538a2c1755b7 \ - --hash=sha256:88e4c8a91b2af5962bfa5ea2447ec6dd357018e86e94c7d14bd8cacbc5b55d81 -PyYAML==3.12 \ - --hash=sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f \ - --hash=sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736 \ - --hash=sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269 \ - --hash=sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8 \ - --hash=sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4 \ - --hash=sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1 \ - --hash=sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab \ - --hash=sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca \ - --hash=sha256:4474f8ea030b5127225b8894d626bb66c01cda098d47a2b0d3429b6700af9fd8 \ - --hash=sha256:326420cbb492172dec84b0f65c80942de6cedb5233c413dd824483989c000608 \ - --hash=sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3 \ - --hash=sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8 \ - --hash=sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6 \ - --hash=sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7 -shutilwhich==1.1.0 \ - --hash=sha256:db1f39c6461e42f630fa617bb8c79090f7711c9ca493e615e43d0610ecb64dc6 -sdnotify==0.3.1 \ - --hash=sha256:e69220d4f6cbb02130f43f929350a80cf51033fde47dcb056fbda71e2dff2d5a -psutil==4.3.1 \ - --hash=sha256:b0c5bf0d2a29a6f18ac22e2d24210730dca458c9f961914289c9e027ccb5ae43 \ - --hash=sha256:fc78c29075e623b6ea1c4a1620a120a1534ee05370b76c0ec96f6d161d79e7a1 \ - --hash=sha256:aa05f44a77ef83773af39446f99e461aa3b6edb7fdabeefdcf06e913d8884d3a \ - --hash=sha256:6b3882eb16f2f40f1da6208a051800abadb1f82a675d9ef6ca7386e1a208b1ad \ - --hash=sha256:cf1be0b16b38f0e2081ff0c81a1a4321c206a824ba6bd51903fdd440abb370b6 \ - --hash=sha256:afa94bed972722882264a4df06176f6b6e6acc6bcebcc3f1db5428c7271dacba \ - --hash=sha256:d2254f518624e6b2262f0f878931faa4bdbe8a77d1f8826564bc4576c6a4f85e \ - --hash=sha256:3b377bc8ba5e62adbc709a90ea07dce2d4addbd6e1cc7acede61ddfa1c66e00a \ - --hash=sha256:38f74182fb9e15cafd0cdf0821098a95cc17301807aed25634a18b66537ba51b \ - --hash=sha256:733210f39e95744da26f2256bc36035fc463b0ae88e91496e97486ba21c63cab \ - --hash=sha256:4690f720054beff4fc66551a6a34512faff328588dca8e2dbed94398b6941112 \ - --hash=sha256:fd9b66edb9f8943eda6b39e7bb9bff8b14aa8d785f5b417d7a0bfa53d4781a7a \ - --hash=sha256:9ab5b62c6571ce545b1c40b9740af81276bd5d94439fd54de07ed59be0ce3f4f \ - --hash=sha256:ad8857923e9bc5802d5559ab5d70c1abc1a7be8e74e779adde883c5391e2061c \ - --hash=sha256:ae20b76cddb3391ea37de5d2aaa1656d6373161bbc8fd868a0ca055194a46e45 \ - --hash=sha256:0613437cc28b8721de92c582d5baf742dfa6dd824c84b578f8c49a60077e969a \ - --hash=sha256:c2031732cd0fb7536af491bb8d8119c9263020a52450f9999c884fd49d346b26 -lmdb==0.89 \ - --hash=sha256:79a0a65c916ede4a50b1c2fd9b93c9141dd8b2cd775d42a283441aff1c9b1e65 \ - --hash=sha256:e58b303b29a8d068ed01699a3e4b7d6061e762e62656cdd0076b51f59a067b1f \ - --hash=sha256:7f8367590ab1815c26b3d3364c66bfa8d3f09640b1d86184024464fa76266933 \ - --hash=sha256:92b1b400a7a69b6f5a7379d3be2b35a76de28076611559adfd5dd6e959a64a31 \ - --hash=sha256:fbb75204a11682ade37cb3f3fd0815c6be1b8ab49eb921b3dede8cbd80dd46b1 \ - --hash=sha256:c0f4720abf97df736a364e01cea3fea979d4cafecdd11d95029829dee24b3784 \ - --hash=sha256:fdef574ba920e0057d16b7bdd90a03809bc74a7f0e729bb8282b5bf6125e3f00 \ - --hash=sha256:b51125bd72966c089f40e8fac455c693943dee2a609269f06c761deac8d1e466 \ - --hash=sha256:3e7575d030ceb010b3c9cf40ec20c5a4e72a064ffa1a3a5c7641fbaa26994562 \ - --hash=sha256:f2aebceb3933904e60e711db8cbaa22437a646e1980e34f7a427eaad6bbd80e5 \ - --hash=sha256:379666d925a95b07a44ceef7200e0fc237701e64af5e8ce9e925a2e77a4f8c83 \ - --hash=sha256:09353405c738a1c8c20c3575efc41d791075a07e15ac357753f57659232d68c0 \ - --hash=sha256:40e275ea1ebb51ec908d42911c28be818f6f8d2eaaefd6283edc396f6e88fe54 \ - --hash=sha256:bb65f0cd24915fe2372d83bd7fefb177d2d331427a5553c8d30a6064efe69136 \ - --hash=sha256:3ecc20afd7d245ef3c76eba6a732d142dbab480eff637ac87caa3599e04a3b81 \ - --hash=sha256:5f54eb1272cda00eca8c77d6aa29175dee0721eb05b1f8575a26dd579c3c4e43 \ - --hash=sha256:406b435e8243b5fdf71313d6d82ce0aeddda4c8c415d5947cc438d46d8747225 \ - --hash=sha256:91679141eae8ce822dc0bfb73bec5b92b8cff00fd11355d76e6cd2bfa5346f49 \ - --hash=sha256:556ad177c77d9f9f82864c50a91240c81b8011f5cc574f3898717ee96e629b84 \ - --hash=sha256:6d3c666713eddb3f8fc0f0b532113122dc4a9282a90d8cd48342d60f238111cf \ - --hash=sha256:d64f302a7b377fa70d62de1e4ad3fd10fe870700ca5bdb788344f80a61b2f58f \ - --hash=sha256:073a64e330dcb9bdd4c9e6a6e7bf8513f8d1cd1fb8ae7a7a3fc09fd22d4c6722 \ - --hash=sha256:677a2fd768c4782684c8f9f203f4f44c867343ef512255879c001c0dc03a14f2 \ - --hash=sha256:8ce5fb0075541f41e3dc911eb45c47438d8ae217ae8da3d8f7d365ec909b8a5d \ - --hash=sha256:650336607e5644ad7c6c4f995faaac0349b409a30cac765a1d0ec899fd20d76d -u-msgpack-python==2.2 \ - --hash=sha256:cdd5a88f2313856a6a9f481f652dab9edd5731059badbb0111280f27249930d7 \ - --hash=sha256:ba121a0cd9bf57b3f8d8a67932e10c51111e390d21153479ebfe17ed48500499 +bitstring==3.1.5 \ + --hash=sha256:c163a86fcef377c314690051885d86b47419e3e1770990c212e16723c1c08faa cbor==1.0.0 \ --hash=sha256:13225a262ddf5615cbd9fd55a76a0d53069d18b07d2e9f19c39e6acb8609bbb6 -py-ubjson==0.8.5 \ - --hash=sha256:e58e99753aaff3ed5d884989a6236bd98a4b0dfdb0e62f9753c20f485691fa9e +cffi==1.8.3 \ + --hash=sha256:d301161bbabdc17d61886e529fb9ea8305cbf2f3f03a86831bd77144c5e9a781 \ + --hash=sha256:1090590fbeb8f369251c822c3e6516b891b6fdd577625d71bab31bacbd304c77 \ + --hash=sha256:24e47ebf130ed26b970bcd2bc2436cdebffb24a1cf4a9cd95c5306b67439f367 \ + --hash=sha256:7cbba4967c04b0940de743c4e3ecb82274ba5568fc100360aeae78d83921b34e \ + --hash=sha256:4ff2b8650bc0fa499d24dbbe85b41faffe2c4c1a4fc241edf075443a5c519591 \ + --hash=sha256:c46eab25dcceb4ef98094a1bc5bd3d6a3fd4429b27e0a53dc9e86b9e7375a2d7 \ + --hash=sha256:2e882a295f17cb13f992bde07647e05af7fe91fa5854d3f10cad8c43401e0522 \ + --hash=sha256:74866d355ea3afe8b33f735ac4d14d485815e6d189a679f4fb2348c0113063fb \ + --hash=sha256:1ee7295bf61ed60eb544970484eeb47a17be64e1596654ab4614a038688e5430 \ + --hash=sha256:0ee962d901fdd5afe53e6c609edca76427cd5b9de6945fd66c5fc0fc23c19017 \ + --hash=sha256:d64b7084a0e97903c779f19605236eddcb4ec27fbe25c16bad3b96c275384325 \ + --hash=sha256:e68f4f39c079ad3819b653fcc2dfa44f234a037473f19d6e7d1685d8de7d24f6 \ + --hash=sha256:7fe6eb09ec7db1c269040cb92e5167fafaa5da747a6400d170f9051c52f08730 \ + --hash=sha256:2d92b3d661211ceac94aca2a4efda4db709174e04c9a30ed12fb8ab78e78672f \ + --hash=sha256:b56289f06c124ed191081df55daeb37b571ade9375d1e18988d0c9819e4a675f \ + --hash=sha256:a7f917d784d51210516ed71b28d4994887be8ed2d99754c1bb49b12aa98bd5fa \ + --hash=sha256:47ea859cd153300778f33220c69cad97cc9c4d3e312a50b01d1521b94789ebfc \ + --hash=sha256:91757b9cfc1b47ffd563ce46090b2494eaec7a1ffd7f41a6f1771dc199f21115 \ + --hash=sha256:d0140dbca2134825b3f38024d4e140a2ad9baeb6ee8f87fd48a442062f417ad0 \ + --hash=sha256:ba1faa81f4e6daa8025e52ed249dcb36ae27ef00047499e1741896688433c146 \ + --hash=sha256:d9c345d966685cd558b7f2d1ec6520cca1823e36c0bad89692808f13bdb5e12f \ + --hash=sha256:9dcc07d7d5b786329088461498523514cd51adab02a3e8ca7f764cad8b9aa50e \ + --hash=sha256:9e466d8dd18ab0ee8def4e2c5df2bf0892957c9276b6eb406d8efcc632f3ded7 \ + --hash=sha256:45445bcd19a01fcd33041cbf1aa997216296d44a32d892b0ae780dc2644258b5 \ + --hash=sha256:6f2802177893c97beeddbacb96bded0db411ccd6b05b020eebcd42d20e2b96d8 \ + --hash=sha256:712b4cc349a56a7d5895791cda291ffba8da407375098f283a6496d2b088f6e7 \ + --hash=sha256:8dafe0d5a19e039a4ee8bc6070d2136e9373ea85fbdd3e2b2cab21f7d60c77e7 \ + --hash=sha256:91517da73b86cbfe1911bc5f1e0c66aa270d0d73b6543be9aa0befa12d9239d3 \ + --hash=sha256:828cdf093ad07b41d8005c8bafd1cd197a9b0aae33fc2f5b52ddbdb4bec765b1 \ + --hash=sha256:6ff29804567502cdde5b6f2efac1caaaae62a0232d21f4d39a2ca744c6a3df7c \ + --hash=sha256:b7a146d7229e0c3e9818c4240eade4430d0af88495037d27085374abef23ff83 \ + --hash=sha256:47e0337a814afd05257ce5bfede318b1dcabcc90e3c50906643e0aa17dab738c \ + --hash=sha256:75348f45112cee84c233a010f404b9369da438e578104dd3b01f1054f9f7519a \ + --hash=sha256:f76b68ba5f90a60a6cc319efd45c46a578ecf15d6d10a1f8055f59da6af77d38 \ + --hash=sha256:f48ec170fa1d6e388a3162af000b76e84455f4c29cd62b08d64f49bf2a98581c \ + --hash=sha256:c321bd46faa7847261b89c0469569530cad5a41976bb6dba8202c0159f476568 +click==6.6 \ + --hash=sha256:cc6a19da8ebff6e7074f731447ef7e112bd23adf3de5c597cf9989f2fd8defe9 +constantly==15.1.0 \ + --hash=sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d \ + --hash=sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35 cryptography==1.5.2 \ --hash=sha256:9e65f4c0ddcd4a7da3cfc1d87a0c3cf735a859c78f5f11d2346f7dfbc31df51b \ --hash=sha256:a7e4a0f46ad767d4083faf31f4304301437f3919017203260620efbfeb72792b \ @@ -126,7 +63,7 @@ cryptography==1.5.2 \ --hash=sha256:3c73d538cb494924929a61376ecd7f802b28b12139546b9775d66ce8071efaf9 \ --hash=sha256:713a68355550423dfb9167ed2365c1ad3aab1644cd7dcaf42afecc1e1d460dc5 \ --hash=sha256:bf90be94e599ab3097f261ca606bad8c75b54eaeaaacfd0818fe6f2616ef9521 \ - --hash=sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ + --hash=sha256:446e9a139c87c09a07c9b252a2336bedffe99821f37332d4fc820d3da90d1738 \ --hash=sha256:f04c00e81d42ec86e0b31a1d91783c3666691a85239f6daecfcda2cbe6c15f28 \ --hash=sha256:5a7b9d10ef04e9cbbf0d9ba6a3502a1cc9176510e89522575d3e338f188eccfe \ --hash=sha256:f2c35caf388e1f503b10c78ced239441f383c2b3d96e1b1c6fe04d56335af84e \ @@ -139,9 +76,79 @@ cryptography==1.5.2 \ --hash=sha256:ef5692b5e44587e92b1231154bec5c7d0a262c75c4be754a6e605de8614145c6 \ --hash=sha256:b634baf73c2b2f0e9c338434531aca3adffef47e78cba909da0ddcc9448f1c84 \ --hash=sha256:eb8875736734e8e870b09be43b17f40472dc189b1c422a952fa8580768204832 -pyOpenSSL==16.1.0 \ - --hash=sha256:13a0d85cb44b2b20117b1ff51b3bce4947972f2f5e5e705a2f9d616457f127ca \ - --hash=sha256:88f7ada2a71daf2c78a4f139b19d57551b4c8be01f53a1cb5c86c2f3bf01355f +enum34==1.1.6 \ + --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \ + --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \ + --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1 \ + --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 +idna==2.1 \ + --hash=sha256:4cfe64db2804351249d8d13ec1f3a2b0be9dc84b409b65c2a646c4d673fa55bc \ + --hash=sha256:f28df695e9bede8a19b18a8e4429b4bad4d664e8e98aff27bc39b630f1ae2b42 \ + --hash=sha256:ed36f281aebf3cd0797f163bb165d84c31507cedd15928b095b1675e2d04c676 +incremental==16.10.1 \ + --hash=sha256:cc00a35f9e9621ddec0a5e7d17271f3f26244873c4aec67eda06f2db8302e04b \ + --hash=sha256:14ad6b720ec47aad6c9caa83e47db1843e2b9b98742da5dda08e16a99f400342 +ipaddress==1.0.17 \ + --hash=sha256:a42764805223838ec9d04eee7fe699083bf5321001792b6076322784c5b994a5 \ + --hash=sha256:3a21c5a15f433710aaa26f1ae174b615973a25182006ae7f9c26de151cd51716 +Jinja2==2.8 \ + --hash=sha256:1cc03ef32b64be19e0a5b54578dd790906a34943fe9102cfdae0d4495bd536b4 \ + --hash=sha256:bc1ff2ff88dbfacefde4ddde471d1417d3b304e8df103a7a9437d47269201bf4 +lmdb==0.92 \ + --hash=sha256:491e40745402a2e46af838bff4ee148c2943d8bde8294f92188c0618ec85bf27 \ + --hash=sha256:87dd3b346d1bc63ba9cae192117897eaadf3bd9b68116f1a5a611d059df6fd6d \ + --hash=sha256:f89a52bcd16d7fde84a9e249a6254b83cc474acea6d3a1f2af7cee1713f6aa1a \ + --hash=sha256:68630b215ca0b9e1c13e7b5368e523ecb720e1d99f36868750ed50656704c689 \ + --hash=sha256:d97c5beaf5b9a06b69fbd5082da0caf3252c3e982256728527d6d3a5ad4953d1 \ + --hash=sha256:796557c030b24a0c76652f74851d3f378d5f7b1b9d85db5324227fc2fb2b2b5f \ + --hash=sha256:a516f75311788077d9d722ab8ec66dc36e4730ff4493e30d9d4d3d222c6e7c35 \ + --hash=sha256:39e591c3d520c31921b6c37658c4ef76567181416c39134a3b83e8abde6fb0e1 \ + --hash=sha256:dc5356b6b13b094ff13b7313c9fe199dfe2ef7a11668991ad040f97106825e33 \ + --hash=sha256:3a5f26bafc0bce5921ec01e13639ea76023cdc299ead234888acfcb23987b5d1 \ + --hash=sha256:c24652c009cfe92f6ff2d837a2474ebb38728ea38c71c15d3ce7058668fb80e0 \ + --hash=sha256:5a85befdd44f088cf9a4ec39cd45c5845fb7bf8b0b21c1ad0b4d15c773225769 \ + --hash=sha256:82cb9743c9f166cc130a67f8f9286ac778676f4c305d6eb85cca178c6fc7aa0b \ + --hash=sha256:27fb9c253ac24a59b913be72174a4f7d155c9ea0b2c6c25903e7e5c06f87fb6c \ + --hash=sha256:d6caa0f578c8e301e1d761bbb8c3f5532bace43cbc80dfa141ff8913a353baad \ + --hash=sha256:819bf3f9d403301c8c988eb572c2a07afdc96291b3fafd46b938975188a37088 \ + --hash=sha256:d08e4aefa3f60c0c48e3df6ca92622fe1bb265b9341d25465b0ee2882d4024e3 \ + --hash=sha256:2b8bd1ff5536426417512fd493719fa0be3319875c37c940ad243835165c73dc \ + --hash=sha256:00fd120160388d22a2dea8750beb8be846dfb439c76a095753d4caa96fa64a70 \ + --hash=sha256:6c557a927763115512bd95d8527ab45095a3d9c6392bd0319d3265e5014102a1 \ + --hash=sha256:596ab7102245831f1398eaa12beb4ca037187121821243794d08b6d6936b0a84 \ + --hash=sha256:3d73238ab0e1f9d8b8ea4916b66f9e91574ec64f844f91b6e9374e894036dc06 +MarkupSafe==0.23 \ + --hash=sha256:a4ec1aff59b95a14b45eb2e23761a0179e98319da5a7eb76b56ea8cdc7b871c3 +mistune==0.7.3 \ + --hash=sha256:ee7447aadcf1962b5af767ff0443dcb0499c16bf73ad36dc99d230e7574571e5 \ + --hash=sha256:21d0e869df3b9189f248e022f1c9763cf9069e1a2f00676f1f1852bd7f98b713 +netaddr==0.7.18 \ + --hash=sha256:cb305179658334eb035860e515f054504e232b832abb4efc51c04bf8a72d3574 \ + --hash=sha256:98c3d6fe831d119785c37af25f032169dd653a0dad86e54b387f4a5b83da4383 \ + --hash=sha256:a1f5c9fcf75ac2579b9995c843dade33009543c04f218ff7c007b3c81695bd19 \ + --hash=sha256:c64c570ac612e20e8b8a6eee72034c924fff9d76c7a46f50a9f919085f1bfbed +pathtools==0.1.2 \ + --hash=sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0 +psutil==4.4.2 \ + --hash=sha256:15aba78f0262d7839702913f5d2ce1e97c89e31456bb26da1a5f9f7d7fe6d336 \ + --hash=sha256:69e30d789c495b781f7cd47c13ee64452c58abfc7132d6dd1b389af312a78239 \ + --hash=sha256:e44d6b758a96539e3e02336430d3f85263d43c470c5bad93572e9b6a86c67f76 \ + --hash=sha256:c2b0d8d1d8b5669b9884d0dd49ccb4094d163858d672d3d13a3fa817bc8a3197 \ + --hash=sha256:10fbb631142a3200623f4ab49f8bf82c32b79b8fe179f6056d01da3dfc589da1 \ + --hash=sha256:e423dd9cb12256c742d1d56ec38bc7d2a7fa09287c82c41e475e68b9f932c2af \ + --hash=sha256:7481f299ae0e966a10cb8dd93a327efd8f51995d9bdc8810dcc65d3b12d856ee \ + --hash=sha256:d96d31d83781c7f3d0df8ccb1cc50650ca84d4722c5070b71ce8f1cc112e02e0 \ + --hash=sha256:1c37e6428f7fe3aeea607f9249986d9bb933bb98133c7919837fd9aac4996b07 \ + --hash=sha256:11a20c0328206dce68f8da771461aeaef9c44811e639216fd935837e758632dc \ + --hash=sha256:642194ebefa573de62406883eb33868917bab2cc2e21b68d551248e194dd0b0a \ + --hash=sha256:c02b9fb5f1f3c857938b26a73b1ca92007e8b0b2fd64693b29300fae0ceaf679 \ + --hash=sha256:6c40dc16b579f645e1804341322364203d0b21045747e62e360fae843d945e20 \ + --hash=sha256:c353ecc62e67bf7c7051c087670d49eae9472f1b30bb1623d667b0cd137e8934 \ + --hash=sha256:7106cb3722235ccb6fe4b18c51f60a548d4b111ec2d209abdcd3998661f4593a \ + --hash=sha256:de1f53fe955dfba562f7791f72517935010a2e88f9caad36917e8c5c03de9051 \ + --hash=sha256:2eb123ca86057ed4f31cfc9880e098ee7a8e19c7ec02b068c45e7559ae7539a6 +py-ubjson==0.8.5 \ + --hash=sha256:e58e99753aaff3ed5d884989a6236bd98a4b0dfdb0e62f9753c20f485691fa9e pyasn1==0.1.9 \ --hash=sha256:61f9d99e3cef65feb1bfe3a2eef7a93eb93819d345bf54bcd42f4e63d5204dae \ --hash=sha256:1802a6dd32045e472a419db1441aecab469d33e0d2749e192abdec52101724af \ @@ -166,9 +173,11 @@ pyasn1-modules==0.0.8 \ --hash=sha256:163d3c689a8f0690c1ec88799b29a34b06f9401e298a4565a2e4fa400445a69c \ --hash=sha256:da2590c2493f26a51f8de53e2805c2eb7112c3b44ec35060e6ea397b92dd1732 \ --hash=sha256:9ff468a0a40a745b046527f173c763e01a774c1ef60d23b7eb90484620a65d9e -service-identity==16.0.0 \ - --hash=sha256:5c63e1532135f7171cb33e8b2f31c758c3ce0654e6eb198573e23d01128678d4 \ - --hash=sha256:0630e222f59f91f3db498be46b1d879ff220955d7bbad719a5cb9ad14e3c3036 +pycparser==2.17 \ + --hash=sha256:0aac31e917c24cb3357f5a4d5566f2cc91a19ca41862f6c3c22dc60a629673b6 +Pygments==2.1.3 \ + --hash=sha256:485602129949b14247e8b124d28af4654dffbd076537c4a9c44a538a2c1755b7 \ + --hash=sha256:88e4c8a91b2af5962bfa5ea2447ec6dd357018e86e94c7d14bd8cacbc5b55d81 PyNaCl==1.0.1 \ --hash=sha256:eb7ba561a8ae2faeeafae38218100f015c4055408af1eda5f9ff7c536cdd3faf \ --hash=sha256:96fe0af92008488c0ad805920ccb7abc6742cfeef173f0c117f2f26a054b33ee \ @@ -183,95 +192,77 @@ PyNaCl==1.0.1 \ --hash=sha256:e40487e3b8d0a16f038970732c3705a89b0a188c065603edd871b6a25a40bf97 \ --hash=sha256:394853427159419c5dcd3d5cd8db2f14592ac3b5215df6ae16613577b21b76e8 \ --hash=sha256:d21d7a7358a85fb9b9ddadfbd1176c40fe199334fe2202881255e77f6d3773f4 -treq==15.1.0 \ - --hash=sha256:1ad1ba89ddc62ae877084b290bd327755b13f6e7bc7076dc4d8e2efb701bfd63 \ - --hash=sha256:425a47d5d52a993d51211028fb6ade252e5fbea094e878bb4b644096a7322de8 -setproctitle==1.1.10 \ - --hash=sha256:6283b7a58477dd8478fbb9e76defb37968ee4ba47b05ec1c053cb39638bd7398 \ - --hash=sha256:6a035eddac62898786aed2c2eee7334c28cfc8106e8eb29fdd117cac56c6cdf0 +pyOpenSSL==16.2.0 \ + --hash=sha256:26ca380ddf272f7556e48064bbcd5bd71f83dfc144f3583501c7ddbd9434ee17 \ + --hash=sha256:7779a3bbb74e79db234af6a08775568c6769b5821faecf6e2f4143edb227516e PyQRCode==1.2.1 \ --hash=sha256:fdbf7634733e56b72e27f9bce46e4550b75a3a2c420414035cae9d9d26b234d5 \ --hash=sha256:1b2812775fa6ff5c527977c4cd2ccb07051ca7d0bc0aecf937a43864abe5eff6 -watchdog==0.8.3 \ - --hash=sha256:7e65882adb7746039b6f3876ee174952f8eaaa34491ba34333ddf1fe35de4162 -argh==0.26.2 \ - --hash=sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3 \ - --hash=sha256:e9535b8c84dc9571a48999094fda7f33e63c3f1b74f3e5f3ac0105a58405bb65 -attrs==16.2.0 \ - --hash=sha256:ce9d6cac4705e5aeaca02d3ff72f0006bf9b0a2f29635ae8dab8262e296f6442 \ - --hash=sha256:136f2ec0f94ec77ff2990830feee965d608cab1e8922370e3abdded383d52001 -cffi==1.7.0 \ - --hash=sha256:e9c79c3f063db325d4957948e6df4d8fae91fdf7c5b1933c40b994623462f687 \ - --hash=sha256:8b57e9a1b73d0592ffc70b6b6d726d7646792d6c5696718491af06244e4be2fe \ - --hash=sha256:75e3300e73a19abe73596e69fcacb6962d7d7d0bbcec6416ac06ebe05d041953 \ - --hash=sha256:8459bcabd7dac17daaae50b1d7e2c112f955b99d072c62e87d708d6c5730b5c8 \ - --hash=sha256:c03060a15dae98800c00dee4f28cf824e178532d87f4676ebe3a01957e53c380 \ - --hash=sha256:06f80ebba6e27b85d4d9727143f8f6b63241898f2625e86edf7eed515049d535 \ - --hash=sha256:822fefe130f4f02405694638dd4963d8020dc4a1b2160293747e3abba7e06a84 \ - --hash=sha256:d546a12ef41394f7140f5cad71f4b8d8f7c00cf54cb852d2796c2d38d4e39807 \ - --hash=sha256:07ea48ed76ab12452fab98d86b8b16ab4cb10d2d1fc5d7bcb73b6a571e008ea8 \ - --hash=sha256:0901aa21b7a7f4eb4b5ff7bb73eafb8ca0a40614110c8dc71cd0cc054f8957a3 \ - --hash=sha256:fc580bcaccdd6765af27885e0751d4f178787f3e52cce4de04ba93bd314c79c1 \ - --hash=sha256:870299a5d4ff88118f68027bbe4f2dbadd6c90b4dd0e0c3c8a368656059367a4 \ - --hash=sha256:08a7f66f69e211f0f63696593f767aeb61ff61fb64eeb510378a2242170dc6dc \ - --hash=sha256:6387faa1f6465b5270aa5dfccfefed730bf0bcf501990e3df48b880ade229418 \ - --hash=sha256:0b7192b918b424fa467a74b49f737ad1dcbd7da33bb789554774b2d2073b896b \ - --hash=sha256:681b75175dda622215b448a768c6f5caf025b7a0de729e4ca17496b979dc67f7 \ - --hash=sha256:69b30b8fd40dbbe9dba98620d0eddf6364128adcabb47b6e7fe8ec1d0610b826 \ - --hash=sha256:53197083750b198568398ca1008916dd8b391ba5c231916c86634ce0b4b8dd8c \ - --hash=sha256:a2e7b20e62095b68ad4a897ec61e9198393ceca18623c1779d376fa09d568b3a \ - --hash=sha256:3350eff45ee63cf8bb6e0698a094d73566fb6b022e966d7bd9cfdf62f66828f0 \ - --hash=sha256:0efa5131175174fb54da6a33335603eaae2bb48cbca7e36586684db46e41558b \ - --hash=sha256:b8767576c35446b7b29c5d899d62e0cebad07f71f8df81679a58f0c5b4bdd8c6 \ - --hash=sha256:e455f3828ebfaa29d8159177918328b7dbc7a3fb96368e53954c95e49254f349 \ - --hash=sha256:b0aa4eb4750f55f3a0601492af02ff33b138fdec3377476f57e1b1b6d47022f3 \ - --hash=sha256:de8acec32abea99cd62908ded38f4c01ce9b9ccf619b8b8100e4fa866762cede \ - --hash=sha256:9151dbec303410e32b6134495067c546a55f9850fe339cd53aaff712958ac586 \ - --hash=sha256:b0adfaa325dfc01b4b49055312c4ac40c2c94371e91e3c9be2b74cefb10b5eef \ - --hash=sha256:d07f62982bac1dcc7fe301f19b6580ec4866d9ed8af1a17137cc544981389a71 \ - --hash=sha256:55933bab6b8458769eb5be5ac76125f2797f614e62a8fd177f3cd656fa758111 \ - --hash=sha256:9f6851f74ad9e2b240de717f67ca04c6481245e1568fa16ea8fd00214e569ac2 \ - --hash=sha256:db669b191e032d30e3416504bc038a787a32f7364482ae9d953782af12a4752a \ - --hash=sha256:ffaed690e336908274c9ebce9dcabba5d29e0975944cb7a6325a1b55ce8fdb99 \ - --hash=sha256:81f11a42f7821516da513a591c91427a21fa75a82e5d857055dc5e0bec02ba5f \ - --hash=sha256:e499d2360f3fa81a01d3252088149e63286b9097b1e074124e28fbe7012518a0 \ - --hash=sha256:afd2f01b290a1f51513e83e12fcbc0e225bef8bf4234fe4964362a3bcb50e161 \ - --hash=sha256:6ed5dd6afd8361f34819c68aaebf9e8fc12b5a5893f91f50c9e50c8886bb60df -enum34==1.1.6 \ - --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \ - --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \ - --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1 \ - --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 -idna==2.1 \ - --hash=sha256:4cfe64db2804351249d8d13ec1f3a2b0be9dc84b409b65c2a646c4d673fa55bc \ - --hash=sha256:f28df695e9bede8a19b18a8e4429b4bad4d664e8e98aff27bc39b630f1ae2b42 \ - --hash=sha256:ed36f281aebf3cd0797f163bb165d84c31507cedd15928b095b1675e2d04c676 -ipaddress==1.0.16 \ - --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ - --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 -MarkupSafe==0.23 \ - --hash=sha256:a4ec1aff59b95a14b45eb2e23761a0179e98319da5a7eb76b56ea8cdc7b871c3 -pathtools==0.1.2 \ - --hash=sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0 -pycparser==2.14 \ - --hash=sha256:7959b4a74abdc27b312fed1c21e6caf9309ce0b29ea86b591fd2e99ecdf27f73 -requests==2.11.0 \ - --hash=sha256:8b9b147f3dff1fc4055ff794ff931f735ed25e87efe667ed7c845a4bafae9b73 \ - --hash=sha256:b2ff053e93ef11ea08b0e596a1618487c4e4c5f1006d7a1706e3671c57dea385 +PyTrie==0.2 \ + --hash=sha256:b272021351efadc6757591aac03ed4794bdfd091122204a4673e94bfb66cc500 +PyYAML==3.12 \ + --hash=sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f \ + --hash=sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736 \ + --hash=sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269 \ + --hash=sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8 \ + --hash=sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4 \ + --hash=sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1 \ + --hash=sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab \ + --hash=sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca \ + --hash=sha256:4474f8ea030b5127225b8894d626bb66c01cda098d47a2b0d3429b6700af9fd8 \ + --hash=sha256:326420cbb492172dec84b0f65c80942de6cedb5233c413dd824483989c000608 \ + --hash=sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3 \ + --hash=sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8 \ + --hash=sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6 \ + --hash=sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7 +requests==2.11.1 \ + --hash=sha256:545c4855cd9d7c12671444326337013766f4eea6068c3f0307fb2dc2696d580e \ + --hash=sha256:5acf980358283faba0b897c73959cecf8b841205bb4b2ad3ef545f46eae1a133 +sdnotify==0.3.1 \ + --hash=sha256:e69220d4f6cbb02130f43f929350a80cf51033fde47dcb056fbda71e2dff2d5a +service_identity==16.0.0 \ + --hash=sha256:5c63e1532135f7171cb33e8b2f31c758c3ce0654e6eb198573e23d01128678d4 \ + --hash=sha256:0630e222f59f91f3db498be46b1d879ff220955d7bbad719a5cb9ad14e3c3036 +setproctitle==1.1.10 \ + --hash=sha256:6283b7a58477dd8478fbb9e76defb37968ee4ba47b05ec1c053cb39638bd7398 \ + --hash=sha256:6a035eddac62898786aed2c2eee7334c28cfc8106e8eb29fdd117cac56c6cdf0 +shutilwhich==1.1.0 \ + --hash=sha256:db1f39c6461e42f630fa617bb8c79090f7711c9ca493e615e43d0610ecb64dc6 six==1.10.0 \ --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \ --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a +treq==15.1.0 \ + --hash=sha256:1ad1ba89ddc62ae877084b290bd327755b13f6e7bc7076dc4d8e2efb701bfd63 \ + --hash=sha256:425a47d5d52a993d51211028fb6ade252e5fbea094e878bb4b644096a7322de8 +Twisted==16.5.0 \ + --hash=sha256:b3594162580b8706073490b44311cfdbb78dca2ddfa3ef791269d924b01db6f0 \ + --hash=sha256:0207d88807482fa670a84926590e163a2a081a29745de34c5a6dc21066abae73 txaio==2.5.1 \ --hash=sha256:a916c98536b8fdfe01e3e8ee6a45d53b88e5c5878980887bdac63cf8aa9697dd \ --hash=sha256:625076477182e2dde78b79d0b2b0d6b3cecb0e24fe0ea7eaf2abd26a4c0dd1de -service_identity==16.0.0 \ - --hash=sha256:5c63e1532135f7171cb33e8b2f31c758c3ce0654e6eb198573e23d01128678d4 \ - --hash=sha256:0630e222f59f91f3db498be46b1d879ff220955d7bbad719a5cb9ad14e3c3036 -bitstring==3.1.5 \ - --hash=sha256:c163a86fcef377c314690051885d86b47419e3e1770990c212e16723c1c08faa -incremental==16.10.0 \ - --hash=sha256:9e399beab7a128daa2836381e3e6843905df61ee328362bfa394131a55ad2284 \ - --hash=sha256:0608d0741ae7ccd4d072f804d62c28180531deef0b882ff2697a737f01754f1d -constantly==15.1.0 \ - --hash=sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d \ - --hash=sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35 +u-msgpack-python==2.3.0 \ + --hash=sha256:d8df6bb0e2a838aa227c39cfd14aa147ab32b3df6871001874e9b9da9ce1760c \ + --hash=sha256:42e4f5f1f1acb37418ac401c40aaa4d6989e0727dbdbac05bfaacbb956448896 +watchdog==0.8.3 \ + --hash=sha256:7e65882adb7746039b6f3876ee174952f8eaaa34491ba34333ddf1fe35de4162 +zope.interface==4.3.2 \ + --hash=sha256:10aba8ab3e011b797656d3a6d0ff8709ee471024f32d6fd056326cca4123e95f \ + --hash=sha256:23eaeb9c59aa41a78cc59b585c80d2eacf96167d575cfd9399f271c967b4b33d \ + --hash=sha256:e727c508a5af11e0bbd690869ed8d3598e6c997e5be66465149f361514a4f3f2 \ + --hash=sha256:ae4963329dc6e78800de72c942422bf641b673fe0708bda7e2a76b96b926121c \ + --hash=sha256:716ee42ea853deb4f6491a0db03e36cf4db82f178ee0f4b3ba433e893d754f16 \ + --hash=sha256:ab357a99d9f00bc43476d70ead78f05d2d7ebc68c8fd8a8a983e2e6e4cb34bfb \ + --hash=sha256:ae3295573d957138585532f2050ca94bed1fdb0fd792f389b1fa6d9cf5df360f \ + --hash=sha256:6d871fb3a7869802fa8ccde0c589e20289e32b30398b24dd46cd1c3bd4dc7382 \ + --hash=sha256:ef296d3f2c26d8b88fd0db57ba13cfcd3df467f333caaeb87f1d31717b14ff87 \ + --hash=sha256:bda6aed260c08c0d1d7c2b100fa3e182e36da064e458af8392acc985fee13644 \ + --hash=sha256:c2fd356ed67d9cd23ed47dddcd118f5b3fba26d4a34eee4bf8974bc80963db98 \ + --hash=sha256:5d78df08d8c571a5ba3c0a95fb9c342487bd63dbe54372224621b556975c5f3e \ + --hash=sha256:7ebc15a8f568162d936da27b8ee01bacf934c3f3758ef5f6a2d4306aeea1337f \ + --hash=sha256:d0e1c5113a10446079f2a8e38e88295b8c34e119d76d30df248bd5b051afdecc \ + --hash=sha256:e270f3a3f3d939ce06e1425e6d70e5d7cc4ccff67e1eaee89c6a6e9d0bdffbfe \ + --hash=sha256:6a0e224a052e3ce27b3a7b1300a24747513f7a507217fcc2a4cb02eb92945cee \ + --hash=sha256:227c4f5908f6ad52b1e1f1748dddb88fb48be32bc4d310eb32032428769dca24 \ + --hash=sha256:cf156d9ec862f1c3a1e56256dc0c3d85048e31f21bc464e03359cba5a1d45250 \ + --hash=sha256:04ae8a1041188faa8de80f9a373e0c39036f4ff5f05b087f45f95e61f5d021e9 \ + --hash=sha256:d5a92d4c10e81d7c2e96a8d6929905b795b46a14fda4b6ec8ec738c02a4f5406 From 454044a91425d3225b25b068d2d509dd6825bc0e Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sun, 30 Oct 2016 16:50:05 +1100 Subject: [PATCH 24/48] update script that does it --- .gitignore | 1 + Makefile | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 2faab08cb..e832617e0 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ node_modules *.ropeproject docs/_build .coverage.* +requirements-latest.txt diff --git a/Makefile b/Makefile index f29de9ff0..cb5965258 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ clean: rm -rf ./.crossbar rm -rf ./_trial_temp rm -rf ./.tox + rm -rf ./vers find . -name "*.db" -exec rm -f {} \; find . -name "*.pyc" -exec rm -f {} \; find . -name "*.log" -exec rm -f {} \; @@ -25,7 +26,7 @@ clean: find . \( -name "*__pycache__" -type d \) -prune -exec rm -rf {} + news: towncrier.ini crossbar/newsfragments/*.* - # this produces a NEWS.md file, 'git rm's crossbar/newsfragmes/* and 'git add's NEWS.md + # this produces a NEWS.md file, 'git rm's crossbar/newsfragments/* and 'git add's NEWS.md # ...which we then use to update docs/pages/ChangeLog.md towncrier cat docs/templates/changelog_preamble.md > docs/pages/ChangeLog.md @@ -39,11 +40,12 @@ docs: # call this in a fresh virtualenv to update our frozen requirements.txt! freeze: clean - pip install --no-cache-dir -r requirements-in.txt - pip freeze -r requirements-in.txt - pip install hashin + virtualenv vers + vers/bin/pip install -r requirements-min.txt + vers/bin/pip freeze > requirements-latest.txt + vers/bin/pip install hashin rm requirements.txt - cat requirements-in.txt | sed -e "s/>=/==/g" | xargs hashin -v > requirements.txt + cat requirements-latest.txt | xargs vers/bin/hashin > requirements.txt wheel: LMDB_FORCE_CFFI=1 SODIUM_INSTALL=bundled pip wheel --require-hashes --wheel-dir ./wheels -r requirements.txt From bce4656156b4f04655f38099a4b577651dc794d5 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sun, 30 Oct 2016 16:50:25 +1100 Subject: [PATCH 25/48] make python -m crossbar work, using the same console script --- crossbar/__main__.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 crossbar/__main__.py diff --git a/crossbar/__main__.py b/crossbar/__main__.py new file mode 100644 index 000000000..a253cc467 --- /dev/null +++ b/crossbar/__main__.py @@ -0,0 +1,37 @@ +##################################################################################### +# +# Copyright (C) Tavendo GmbH +# +# Unless a separate license agreement exists between you and Tavendo GmbH (e.g. you +# have purchased a commercial license), the license terms below apply. +# +# Should you enter into a separate license agreement after having received a copy of +# this software, then the terms of such license agreement replace the terms below at +# the time at which such license agreement becomes effective. +# +# In case a separate license agreement ends, and such agreement ends without being +# replaced by another separate license agreement, the license terms below apply +# from the time at which said agreement ends. +# +# LICENSE TERMS +# +# This program is free software: you can redistribute it and/or modify it under the +# terms of the GNU Affero General Public License, version 3, as published by the +# Free Software Foundation. This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Affero General Public License Version 3 for more details. +# +# You should have received a copy of the GNU Affero General Public license along +# with this program. If not, see . +# +##################################################################################### + +if __name__ == '__main__': + from pkg_resources import load_entry_point + import sys + + sys.exit( + load_entry_point('crossbar', 'console_scripts', 'crossbar')() + ) From b9b554c21c3d3943808dc0814cb0302f44e5d66a Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sun, 30 Oct 2016 16:51:09 +1100 Subject: [PATCH 26/48] add more to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index e832617e0..d2cbe1191 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,5 @@ node_modules docs/_build .coverage.* requirements-latest.txt +vers/ +.DS_Store From 1ba3705aaf4ded69436780b6065d8232cc437b98 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sun, 30 Oct 2016 16:52:04 +1100 Subject: [PATCH 27/48] fix setup --- MANIFEST.in | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 4b04aa5eb..3f68da524 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,7 +2,7 @@ include COPYRIGHT include LICENSE include LICENSE-FOR-API include requirements.txt -include requirements-in.txt +include requirements-min.txt include requirements-dev.txt recursive-include crossbar/web * recursive-include crossbar/templates * diff --git a/setup.py b/setup.py index 2a1ac7e23..ea7935854 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ ] } -with open('requirements-in.txt') as f: +with open('requirements-min.txt') as f: for line in f.read().splitlines(): line = line.strip() if not line.startswith('#'): From 997366fada35baa4ce380fc270babe32005e57f3 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sun, 30 Oct 2016 17:07:54 +1100 Subject: [PATCH 28/48] fix pinned tox installs --- Makefile | 2 +- requirements.txt | 4 ++++ tox.ini | 8 +++----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index cb5965258..46216acc5 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ docs: freeze: clean virtualenv vers vers/bin/pip install -r requirements-min.txt - vers/bin/pip freeze > requirements-latest.txt + vers/bin/pip freeze --all | grep -v -e "wheel" -e "pip" -e "distribute" > requirements-latest.txt vers/bin/pip install hashin rm requirements.txt cat requirements-latest.txt | xargs vers/bin/hashin > requirements.txt diff --git a/requirements.txt b/requirements.txt index 349a4bc69..80f78cf1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -226,6 +226,10 @@ service_identity==16.0.0 \ setproctitle==1.1.10 \ --hash=sha256:6283b7a58477dd8478fbb9e76defb37968ee4ba47b05ec1c053cb39638bd7398 \ --hash=sha256:6a035eddac62898786aed2c2eee7334c28cfc8106e8eb29fdd117cac56c6cdf0 +setuptools==28.7.1 \ + --hash=sha256:5bd5d77813a5b8e30ad84161b3b3262e2b6a6cddc22568d88c5b7fedb404b762 \ + --hash=sha256:6aae26f13411c3102fc915db6e44b62a97f76a7ba7e963daabf78c57240ac166 \ + --hash=sha256:92ac0bcad9a513c940f6b70169d159d30eabc6be25b979f0a6f6401c9585443c shutilwhich==1.1.0 \ --hash=sha256:db1f39c6461e42f630fa617bb8c79090f7711c9ca493e615e43d0610ecb64dc6 six==1.10.0 \ diff --git a/tox.ini b/tox.ini index f532c1781..fceacf737 100644 --- a/tox.ini +++ b/tox.ini @@ -35,8 +35,6 @@ commands= whitelist_externals = sh changedir=tests deps = - #pypy: cryptography<1.0 - #pypy: pynacl<1.0 mock coverage commands = @@ -44,14 +42,14 @@ commands = sh -c "which coverage" python -V pip --version + pinned: pip install --ignore-installed --require-hashes -r {toxinidir}/requirements.txt + python -c "import setuptools; print('setuptools-%s' % setuptools.__version__)" coverage --version {envbindir}/trial --version - pinned: pip install --ignore-installed --require-hashes -r {toxinidir}/requirements.txt + coverage run --rcfile={toxinidir}/.coveragerc -m twisted.trial {posargs:crossbar} setenv = COVERAGE_PROCESS_START = {toxinidir}/.coveragerc COVERAGE_FILE = {toxinidir}/.coverage - #LMDB_FORCE_CFFI=1 - #SODIUM_INSTALL=bundled {py27,py33,py34,py35}: CB_FULLTESTS = 1 From 60e014ebd4b2d1d537434665c231fdc34f784e79 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 5 Nov 2016 15:53:53 +1100 Subject: [PATCH 29/48] some more tests --- crossbar/_log_categories.py | 1 + crossbar/adapter/mqtt/test/test_tx.py | 92 ++++++++++++++++++++++++++- crossbar/adapter/mqtt/tx.py | 6 +- 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/crossbar/_log_categories.py b/crossbar/_log_categories.py index 3a8d04b3e..1c02049aa 100644 --- a/crossbar/_log_categories.py +++ b/crossbar/_log_categories.py @@ -77,6 +77,7 @@ "MQ301": "Got a PubREC for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", "MQ302": "Got a PubREC for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", "MQ302": "Got a PubCOMP for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", + "MQ303": "Got a non-allowed QoS value in the publish queue, dropping it.", "MQ400": "MQTT client '{client_id}' timed out after recieving no full packets for {seconds}", "MQ401": "Protocol violation from '{client_id}', terminating connection: {error}", "MQ402": "Got a packet ('{packet_id}') from '{client_id}' that is invalid for a server, terminating connection", diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index 6261ad0c1..23849dfe6 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -36,7 +36,7 @@ from binascii import unhexlify from crossbar.adapter.mqtt.tx import ( - MQTTServerTwistedProtocol, AwaitingACK, Session) + MQTTServerTwistedProtocol, AwaitingACK, Session, Message) from crossbar.adapter.mqtt.protocol import ( MQTTParser, client_packet_handlers, P_CONNACK) from crossbar.adapter.mqtt._events import ( @@ -1754,3 +1754,93 @@ def test_qos_2_resent_on_disconnect_pubcomp(self): self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) self.assertEqual(len(sessions[u"test123"].queued_messages), 0) self.assertFalse(t2.disconnecting) + + def test_non_allowed_qos_not_queued(self): + """ + A non-QoS 0, 1, or 2 message will be rejected by the publish layer. + """ + got_packets = [] + + h = BasicHandler() + sessions, r, t, p, cp = make_test_items(h) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + ) + + for x in iterbytes(data): + p.dataReceived(x) + + # No queued messages + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + + # Connect has happened + events = cp.data_received(t.value()) + t.clear() + self.assertFalse(t.disconnecting) + self.assertIsInstance(events[0], ConnACK) + + # WAMP layer calls send_publish w/ invalid QoS + with self.assertRaises(ValueError): + p.send_publish(u"hello", 5, b'some bytes') + + # Nothing will be sent or queued + self.assertEqual(t.value(), b'') + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + + # Advance the clock + r.advance(0.1) + + # Still nothing + self.assertEqual(t.value(), b'') + + def test_non_allowed_qos_in_queue_dropped(self): + """ + If a non-QoS 0, 1, or 2 message gets into the queue, it will be + dropped. + """ + got_packets = [] + + h = BasicHandler() + sessions, r, t, p, cp = make_test_items(h) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=True)).serialise() + ) + + for x in iterbytes(data): + p.dataReceived(x) + + # Add a queued message + sessions[u"test123"].queued_messages.append(Message( + topic=u"foo", body=b"bar", qos=3)) + + # Connect has happened + events = cp.data_received(t.value()) + t.clear() + self.assertFalse(t.disconnecting) + self.assertIsInstance(events[0], ConnACK) + + # Nothing is sent, one is queued + self.assertEqual(t.value(), b'') + self.assertEqual(len(sessions[u"test123"].queued_messages), 1) + + with LogCapturer("trace") as logs: + # Flush the saved messages + p._flush_saved_messages() + + # We got the warning + logs = logs.get_category("MQ303") + self.assertEqual(len(logs), 1) + + # Nothing queued + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + + # Nothing sent + self.assertEqual(t.value(), b'') + + +class OutOfOrderAckTests(TestCase): + pass diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index 788286a24..070aa04e5 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -144,6 +144,9 @@ def connectionLost(self, reason): def send_publish(self, topic, qos, body): + if not qos in [0, 1, 2]: + raise ValueError("QoS must be [0, 1, 2]") + self.session.queued_messages.append(Message(topic=topic, qos=qos, body=body)) if not self._flush_publishes and self.transport.connected: self._flush_publishes = self._reactor.callLater(0, self._flush_saved_messages) @@ -177,7 +180,8 @@ def _send_publish(self, topic, qos, body): self.session._in_flight_packet_ids.add(packet_id) else: - raise ValueError("QoS must be [0, 1, 2]") + self.log.warn(log_category="MQ303") + return self._send_packet(publish) From 103bc0d60f5b75c2a854ee87e79e1c4e1bbfc6d2 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 5 Nov 2016 21:59:25 +1100 Subject: [PATCH 30/48] first round of interop testing --- crossbar/adapter/mqtt/test/interop.py | 162 ++++++++++++++++ crossbar/adapter/mqtt/test/interop_tests.py | 205 ++++++++++++++++++++ 2 files changed, 367 insertions(+) create mode 100644 crossbar/adapter/mqtt/test/interop.py create mode 100644 crossbar/adapter/mqtt/test/interop_tests.py diff --git a/crossbar/adapter/mqtt/test/interop.py b/crossbar/adapter/mqtt/test/interop.py new file mode 100644 index 000000000..ea64d216f --- /dev/null +++ b/crossbar/adapter/mqtt/test/interop.py @@ -0,0 +1,162 @@ +from __future__ import print_function + +import click +import attr +import sys + +from collections import deque +from texttable import Texttable + +from twisted.internet.protocol import Protocol, ClientFactory +from twisted.logger import globalLogBeginner, textFileLogObserver +from crossbar.adapter.mqtt.protocol import MQTTClientParser + + +@attr.s +class Frame(object): + send = attr.ib() + data = attr.ib() + +class ConnectionLoss(object): + send = False + data = b"" + + +@attr.s +class Result(object): + name = attr.ib() + success = attr.ib() + reason = attr.ib() + transcript = attr.ib() + + +@click.command() +@click.option("--host") +@click.option("--port") +def run(host, port): + + #globalLogBeginner.beginLoggingTo([textFileLogObserver(sys.stdout)]) + + port = int(port) + + from . import interop_tests + test_names = [x for x in dir(interop_tests) if x.startswith("test_")] + + tests = [getattr(interop_tests, test_name) for test_name in test_names] + + results = [] + with click.progressbar(tests, label="Running interop tests...") as _tests: + for test in _tests: + results.append(test(host, port)) + + fmt_results = [] + for r in results: + fmt_results.append((r.name, + "True" if r.success else "False", r.reason, r.transcript)) + + t = Texttable() + t.set_cols_width([20, 10, 80, 60]) + rows = [["Name", "Successful", "Reason", "Client Transcript"]] + rows.extend(fmt_results) + t.add_rows(rows) + print(t.draw(), file=sys.__stdout__) + + failures = [] + for x in results: + if not x.success: + failures.append(False) + + if failures: + sys.exit(len(failures)) + sys.exit(0) + + + + +class ReplayProtocol(Protocol): + + def __init__(self, factory): + self.factory = factory + self._record = deque(self.factory.record) + self._buffer = b"" + self._waiting_for_nothing = None + self._client = MQTTClientParser() + + def connectionMade(self): + + if self._record[0].send: + self.transport.write(self._record.popleft().data) + + def dataReceived(self, data): + self.factory._timer.reset(5) + self._buffer = self._buffer + data + + self.factory.client_transcript.extend(self._client.data_received(data)) + + if self._waiting_for_nothing: + self.factory.reason = "Got unexpected data " + repr(self._buffer) + self.factory.success = False + self.factory.reactor.stop() + return + + if len(self._record) > 0 and len(self._buffer) == len(self._record[0].data): + reading = self._record.popleft() + + if self._buffer == reading.data: + pass + else: + self.factory.success = False + self.factory.reason = (self._buffer, reading.data) + self.factory.reactor.stop() + return + + self._buffer = b'' + + if len(self._record) > 0: + if self._record[0].send: + self.transport.write(self._record.popleft().data) + + # Then if we are supposed to wait... + if isinstance(self._record[0], Frame) and self._record[0].send is False and self._record[0].data == b"": + def wait(): + self._waiting_for_nothing = None + self.dataReceived(b"") + self._waiting_for_nothing = self.factory.reactor.callLater(2, wait) + return + + def connectionLost(self, reason): + if self.factory.reactor.running: + if self._record and isinstance(self._record[0], ConnectionLoss): + self.factory.success = True + else: + self.factory.success = False + self.factory.reason = "Premature disconnection" + self.factory.reactor.stop() + + +@attr.s +class ReplayClientFactory(ClientFactory): + reactor = attr.ib() + record = attr.ib() + success = attr.ib(default=None) + reason = attr.ib(default=None) + protocol = ReplayProtocol + noisy = False + + + def buildProtocol(self, addr): + + self.client_transcript = [] + + p = self.protocol(self) + + def disconnect(): + self.reason = "Timeout (buffer was " + repr(p._buffer) + ", remaining assertions were " + repr(p._record) + ")" + self.reactor.stop() + + self._timer = self.reactor.callLater(5, disconnect) + return p + + +if __name__ == "__main__": + run() diff --git a/crossbar/adapter/mqtt/test/interop_tests.py b/crossbar/adapter/mqtt/test/interop_tests.py new file mode 100644 index 000000000..ad3e31824 --- /dev/null +++ b/crossbar/adapter/mqtt/test/interop_tests.py @@ -0,0 +1,205 @@ + +from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.internet.selectreactor import SelectReactor + +from .interop import Result, ReplayClientFactory, Frame, ConnectionLoss +from crossbar.adapter.mqtt._events import ( + Connect, ConnectFlags, ConnACK, + Publish, PubACK, PubREL, PubREC, PubCOMP, + Subscribe, SubscriptionTopicRequest, SubACK, + Disconnect) + +def test_connect(host, port): + record = [ + Frame( + send=True, + data=Connect(client_id=u"test_cleanconnect", + flags=ConnectFlags(clean_session=True)).serialise()), + Frame( + send=False, + data=ConnACK(session_present=False, return_code=0).serialise()), + Frame( + send=True, + data=Disconnect().serialise()), + ConnectionLoss(), + ] + + r = SelectReactor() + f = ReplayClientFactory(r, record) + e = TCP4ClientEndpoint(r, host, port) + e.connect(f) + r.run() + + return Result("connect", f.success, f.reason, f.client_transcript) + + +def test_quirks_mode_connect(host, port): + record = [ + Frame( + send=True, + data=b"\x10\x15\x00\x04MQTT\x04\x02\x00x\x00\x07testqrk\x00\x00"), + Frame( + send=False, + data=ConnACK(session_present=False, return_code=0).serialise()), + Frame( + send=True, + data=Disconnect().serialise()), + ConnectionLoss(), + ] + + r = SelectReactor() + f = ReplayClientFactory(r, record) + e = TCP4ClientEndpoint(r, host, port) + e.connect(f) + r.run() + + return Result("connect_quirks", f.success, f.reason, f.client_transcript) + + +def test_reserved_packet_15(host, port): + record = [ + Frame( + send=True, + data=Connect(client_id=u"test_reserved15", + flags=ConnectFlags(clean_session=True)).serialise()), + Frame( + send=False, + data=ConnACK(session_present=False, return_code=0).serialise()), + Frame( + send=True, + # v pkt 15 right here + data=b"\xf0\x13\x00\x04MQTT\x04\x02\x00\x02\x00\x07test123"), + ConnectionLoss() + ] + + r = SelectReactor() + f = ReplayClientFactory(r, record) + e = TCP4ClientEndpoint(r, host, port) + e.connect(f) + r.run() + + return Result("reserved_pkt15", f.success, f.reason, f.client_transcript) + + +def test_reserved_packet_0(host, port): + record = [ + Frame( + send=True, + data=Connect(client_id=u"test_reserved0", + flags=ConnectFlags(clean_session=True)).serialise()), + Frame( + send=False, + data=ConnACK(session_present=False, return_code=0).serialise()), + Frame( + send=True, + # v pkt 0 right here + data=b"\x00\x13\x00\x04MQTT\x04\x02\x00\x02\x00\x07test123"), + ConnectionLoss() + ] + + r = SelectReactor() + f = ReplayClientFactory(r, record) + e = TCP4ClientEndpoint(r, host, port) + e.connect(f) + r.run() + + return Result("reserved_pkt0", f.success, f.reason, f.client_transcript) + + +def test_uninvited_puback(host, port): + record = [ + Frame( + send=True, + data=Connect(client_id=u"test_puback", + flags=ConnectFlags(clean_session=True)).serialise()), + Frame( + send=False, + data=ConnACK(session_present=False, return_code=0).serialise()), + Frame( + send=True, + data=PubACK(packet_identifier=1234).serialise()), + Frame( + send=False, + data=b""), + Frame( + send=True, + data=Disconnect().serialise()), + ConnectionLoss(), + ] + + r = SelectReactor() + f = ReplayClientFactory(r, record) + e = TCP4ClientEndpoint(r, host, port) + e.connect(f) + r.run() + + return Result("uninvited_puback", f.success, f.reason, f.client_transcript) + + +def test_uninvited_pubrel(host, port): + record = [ + Frame( + send=True, + data=Connect(client_id=u"test_pubrel", + flags=ConnectFlags(clean_session=True)).serialise()), + Frame( + send=False, + data=ConnACK(session_present=False, return_code=0).serialise()), + Frame( + send=True, + data=PubREL(packet_identifier=1234).serialise()), + Frame( + send=False, + data=PubCOMP(packet_identifier=1234).serialise()), + Frame( + send=True, + data=Disconnect().serialise()), + ConnectionLoss(), + ] + + r = SelectReactor() + f = ReplayClientFactory(r, record) + e = TCP4ClientEndpoint(r, host, port) + e.connect(f) + r.run() + + return Result("uninvited_pubrel", f.success, f.reason, f.client_transcript) + + +def test_self_subscribe(host, port): + record = [ + Frame( + send=True, + data=Connect(client_id=u"test_selfsub", + flags=ConnectFlags(clean_session=True)).serialise()), + Frame( + send=False, + data=ConnACK(session_present=False, return_code=0).serialise()), + Frame( + send=True, + data=Subscribe(packet_identifier=1234, + topic_requests=[SubscriptionTopicRequest(u"foo", 2)]).serialise()), + Frame( + send=False, + data=SubACK(packet_identifier=1234, return_codes=[2]).serialise()), + Frame( + send=True, + data=Publish(duplicate=False, qos_level=0, topic_name=u"foo", + payload=b"abc", retain=False).serialise()), + Frame( + send=False, + data=Publish(duplicate=False, qos_level=0, topic_name=u"foo", + payload=b"abc", retain=False).serialise()), + Frame( + send=True, + data=Disconnect().serialise()), + ConnectionLoss(), + ] + + r = SelectReactor() + f = ReplayClientFactory(r, record) + e = TCP4ClientEndpoint(r, host, port) + e.connect(f) + r.run() + + return Result("self_subscribe", f.success, f.reason, f.client_transcript) From fb46572c597d6354f23a920d2e1579e2d52fafe2 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 5 Nov 2016 21:59:46 +1100 Subject: [PATCH 31/48] disconnect, move around some stuff --- crossbar/_log_categories.py | 1 + crossbar/adapter/mqtt/_events.py | 19 ++++++++++++++++++- crossbar/adapter/mqtt/protocol.py | 5 +++++ crossbar/adapter/mqtt/test/test_tx.py | 7 +------ crossbar/adapter/mqtt/tx.py | 19 +++++++++++++++---- 5 files changed, 40 insertions(+), 11 deletions(-) diff --git a/crossbar/_log_categories.py b/crossbar/_log_categories.py index 1c02049aa..8212e8388 100644 --- a/crossbar/_log_categories.py +++ b/crossbar/_log_categories.py @@ -73,6 +73,7 @@ "MQ201": "Received a QoS 0 Publish from '{client_id}'", "MQ202": "Received a QoS 1 Publish from '{client_id}'", "MQ203": "Received a QoS 2 Publish from '{client_id}'", + "MQ204": "Received a Disconnect from '{client_id}', closing connection", "MQ300": "Got a PubACK for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", "MQ301": "Got a PubREC for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", "MQ302": "Got a PubREC for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", diff --git a/crossbar/adapter/mqtt/_events.py b/crossbar/adapter/mqtt/_events.py index 358f81222..137aaff6c 100644 --- a/crossbar/adapter/mqtt/_events.py +++ b/crossbar/adapter/mqtt/_events.py @@ -47,6 +47,22 @@ class Failure(object): reason = attr.ib(default=None) +@attr.s +class Disconnect(object): + def serialise(self): + """ + Assemble this into an on-wire message. + """ + return build_header(14, (False, False, False, False), 0) + + @classmethod + def deserialise(cls, flags, data): + if flags != (False, False, False, False): + raise ParseFailure(cls, "Bad flags") + + return cls() + + @attr.s class PingRESP(object): def serialise(self): @@ -297,8 +313,9 @@ class Publish(object): qos_level = attr.ib(validator=instance_of(int)) retain = attr.ib(validator=instance_of(bool)) topic_name = attr.ib(validator=instance_of(unicode)) - packet_identifier = attr.ib(validator=optional(instance_of(int))) payload = attr.ib(validator=instance_of(bytes)) + packet_identifier = attr.ib(validator=optional(instance_of(int)), + default=None) def serialise(self): """ diff --git a/crossbar/adapter/mqtt/protocol.py b/crossbar/adapter/mqtt/protocol.py index 41a1af4cf..ad93eeb14 100644 --- a/crossbar/adapter/mqtt/protocol.py +++ b/crossbar/adapter/mqtt/protocol.py @@ -227,3 +227,8 @@ def data_received(self, data): self._packet_count += 1 else: return events + + +class MQTTClientParser(MQTTParser): + _first_pkt = P_CONNACK + _packet_handlers = client_packet_handlers diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index 23849dfe6..e846f5612 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -38,7 +38,7 @@ from crossbar.adapter.mqtt.tx import ( MQTTServerTwistedProtocol, AwaitingACK, Session, Message) from crossbar.adapter.mqtt.protocol import ( - MQTTParser, client_packet_handlers, P_CONNACK) + MQTTParser, MQTTClientParser, client_packet_handlers, P_CONNACK) from crossbar.adapter.mqtt._events import ( Connect, ConnectFlags, ConnACK, SubACK, Subscribe, @@ -54,11 +54,6 @@ from twisted.internet.defer import Deferred, succeed, inlineCallbacks -class MQTTClientParser(MQTTParser): - _first_pkt = P_CONNACK - _packet_handlers = client_packet_handlers - - @attr.s class BasicHandler(object): _connect_code = attr.ib(default=0) diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index 070aa04e5..54564b3d9 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -47,6 +47,7 @@ Unsubscribe, UnsubACK, Publish, PubACK, PubREC, PubREL, PubCOMP, PingREQ, PingRESP, + Disconnect, ) from twisted.internet.protocol import Protocol @@ -488,18 +489,21 @@ def _handle_events(self, events): self.transport.loseConnection() returnValue(None) - # MQTT-4.3.3-1: Send back a PubREL self.session._publishes_awaiting_ack[event.packet_identifier].stage = 1 - resp = PubREL(packet_identifier=event.packet_identifier) - self._send_packet(resp) - else: self.log.warn( log_category="MQ301", client_id=self.session.client_id, pub_id=event.packet_identifier) + # MQTT-4.3.3-1: MUST send back a PubREL -- even if it's not an + # ID we know about, apparently, according to Mosquitto and + # ActiveMQ. + resp = PubREL(packet_identifier=event.packet_identifier) + self._send_packet(resp) + elif isinstance(event, PubREL): + # Should check if it is valid here resp = PubCOMP(packet_identifier=event.packet_identifier) self._send_packet(resp) continue @@ -529,6 +533,13 @@ def _handle_events(self, events): log_category="MQ302", client_id=self.session.client_id, pub_id=event.packet_identifier) + elif isinstance(event, Disconnect): + # TODO: get rid of some will messages + + # 3.14.4 -- we can close it if we want to + self.transport.loseConnection() + returnValue(None) + else: if isinstance(event, Failure): self.log.error( From 563b753e084c51e45d4705a1686e1e2f24e96d3c Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 5 Nov 2016 22:37:07 +1100 Subject: [PATCH 32/48] we need to obviously respond a bit differently. --- crossbar/adapter/mqtt/test/interop.py | 76 ++++++---- crossbar/adapter/mqtt/test/interop_tests.py | 151 ++++++++++++++++---- 2 files changed, 173 insertions(+), 54 deletions(-) diff --git a/crossbar/adapter/mqtt/test/interop.py b/crossbar/adapter/mqtt/test/interop.py index ea64d216f..20e172bc2 100644 --- a/crossbar/adapter/mqtt/test/interop.py +++ b/crossbar/adapter/mqtt/test/interop.py @@ -78,51 +78,69 @@ class ReplayProtocol(Protocol): def __init__(self, factory): self.factory = factory self._record = deque(self.factory.record) - self._buffer = b"" self._waiting_for_nothing = None self._client = MQTTClientParser() def connectionMade(self): if self._record[0].send: - self.transport.write(self._record.popleft().data) + to_send = self._record.popleft() + if isinstance(to_send.data, bytes): + self.transport.write(to_send.data) + else: + self.transport.write(to_send.data.serialise()) def dataReceived(self, data): - self.factory._timer.reset(5) - self._buffer = self._buffer + data + self.factory._timer.reset(7) - self.factory.client_transcript.extend(self._client.data_received(data)) + got_data = self._client.data_received(data) + self.factory.client_transcript.extend(got_data) if self._waiting_for_nothing: - self.factory.reason = "Got unexpected data " + repr(self._buffer) - self.factory.success = False - self.factory.reactor.stop() - return - - if len(self._record) > 0 and len(self._buffer) == len(self._record[0].data): - reading = self._record.popleft() - - if self._buffer == reading.data: - pass + if data == b"": + got_data.append(b"") + self._waiting_for_nothing = None else: + self.factory.reason = "Got unexpected data " + repr(got_data) self.factory.success = False - self.factory.reason = (self._buffer, reading.data) self.factory.reactor.stop() return - self._buffer = b'' + if len(self._record) > 0 and got_data: + for x in got_data: + reading = self._record.popleft() + + if x == reading.data: + pass + elif isinstance(reading.data, list) and x in reading.data: + reading.data.remove(x) + else: + self.factory.success = False + self.factory.reason = (x, reading.data) + self.factory.reactor.stop() + return - if len(self._record) > 0: - if self._record[0].send: - self.transport.write(self._record.popleft().data) + if len(self._record) > 0: + while len(self._record) > 0 and self._record[0].send: + to_send = self._record.popleft() + if isinstance(to_send.data, bytes): + self.transport.write(to_send.data) + else: + self.transport.write(to_send.data.serialise()) + + if isinstance(reading.data, list): + if reading.data: + self._record.appendleft(reading) + + if len(self._record) > 0: + + # Then if we are supposed to wait... + if isinstance(self._record[0], Frame) and self._record[0].send is False and self._record[0].data == b"": + def wait(): + self.dataReceived(b"") + self._waiting_for_nothing = self.factory.reactor.callLater(2, wait) + return - # Then if we are supposed to wait... - if isinstance(self._record[0], Frame) and self._record[0].send is False and self._record[0].data == b"": - def wait(): - self._waiting_for_nothing = None - self.dataReceived(b"") - self._waiting_for_nothing = self.factory.reactor.callLater(2, wait) - return def connectionLost(self, reason): if self.factory.reactor.running: @@ -151,10 +169,10 @@ def buildProtocol(self, addr): p = self.protocol(self) def disconnect(): - self.reason = "Timeout (buffer was " + repr(p._buffer) + ", remaining assertions were " + repr(p._record) + ")" + self.reason = "Timeout (remaining assertions were " + repr(p._record) + ")" self.reactor.stop() - self._timer = self.reactor.callLater(5, disconnect) + self._timer = self.reactor.callLater(7, disconnect) return p diff --git a/crossbar/adapter/mqtt/test/interop_tests.py b/crossbar/adapter/mqtt/test/interop_tests.py index ad3e31824..72ed74899 100644 --- a/crossbar/adapter/mqtt/test/interop_tests.py +++ b/crossbar/adapter/mqtt/test/interop_tests.py @@ -14,13 +14,13 @@ def test_connect(host, port): Frame( send=True, data=Connect(client_id=u"test_cleanconnect", - flags=ConnectFlags(clean_session=True)).serialise()), + flags=ConnectFlags(clean_session=True))), Frame( send=False, - data=ConnACK(session_present=False, return_code=0).serialise()), + data=ConnACK(session_present=False, return_code=0)), Frame( send=True, - data=Disconnect().serialise()), + data=Disconnect()), ConnectionLoss(), ] @@ -40,10 +40,10 @@ def test_quirks_mode_connect(host, port): data=b"\x10\x15\x00\x04MQTT\x04\x02\x00x\x00\x07testqrk\x00\x00"), Frame( send=False, - data=ConnACK(session_present=False, return_code=0).serialise()), + data=ConnACK(session_present=False, return_code=0)), Frame( send=True, - data=Disconnect().serialise()), + data=Disconnect()), ConnectionLoss(), ] @@ -61,10 +61,10 @@ def test_reserved_packet_15(host, port): Frame( send=True, data=Connect(client_id=u"test_reserved15", - flags=ConnectFlags(clean_session=True)).serialise()), + flags=ConnectFlags(clean_session=True))), Frame( send=False, - data=ConnACK(session_present=False, return_code=0).serialise()), + data=ConnACK(session_present=False, return_code=0)), Frame( send=True, # v pkt 15 right here @@ -86,10 +86,10 @@ def test_reserved_packet_0(host, port): Frame( send=True, data=Connect(client_id=u"test_reserved0", - flags=ConnectFlags(clean_session=True)).serialise()), + flags=ConnectFlags(clean_session=True))), Frame( send=False, - data=ConnACK(session_present=False, return_code=0).serialise()), + data=ConnACK(session_present=False, return_code=0)), Frame( send=True, # v pkt 0 right here @@ -111,19 +111,19 @@ def test_uninvited_puback(host, port): Frame( send=True, data=Connect(client_id=u"test_puback", - flags=ConnectFlags(clean_session=True)).serialise()), + flags=ConnectFlags(clean_session=True))), Frame( send=False, - data=ConnACK(session_present=False, return_code=0).serialise()), + data=ConnACK(session_present=False, return_code=0)), Frame( send=True, - data=PubACK(packet_identifier=1234).serialise()), + data=PubACK(packet_identifier=1234)), Frame( send=False, data=b""), Frame( send=True, - data=Disconnect().serialise()), + data=Disconnect()), ConnectionLoss(), ] @@ -141,19 +141,19 @@ def test_uninvited_pubrel(host, port): Frame( send=True, data=Connect(client_id=u"test_pubrel", - flags=ConnectFlags(clean_session=True)).serialise()), + flags=ConnectFlags(clean_session=True))), Frame( send=False, - data=ConnACK(session_present=False, return_code=0).serialise()), + data=ConnACK(session_present=False, return_code=0)), Frame( send=True, - data=PubREL(packet_identifier=1234).serialise()), + data=PubREL(packet_identifier=1234)), Frame( send=False, - data=PubCOMP(packet_identifier=1234).serialise()), + data=PubCOMP(packet_identifier=1234)), Frame( send=True, - data=Disconnect().serialise()), + data=Disconnect()), ConnectionLoss(), ] @@ -171,28 +171,28 @@ def test_self_subscribe(host, port): Frame( send=True, data=Connect(client_id=u"test_selfsub", - flags=ConnectFlags(clean_session=True)).serialise()), + flags=ConnectFlags(clean_session=True))), Frame( send=False, - data=ConnACK(session_present=False, return_code=0).serialise()), + data=ConnACK(session_present=False, return_code=0)), Frame( send=True, data=Subscribe(packet_identifier=1234, - topic_requests=[SubscriptionTopicRequest(u"foo", 2)]).serialise()), + topic_requests=[SubscriptionTopicRequest(u"foo", 2)])), Frame( send=False, - data=SubACK(packet_identifier=1234, return_codes=[2]).serialise()), + data=SubACK(packet_identifier=1234, return_codes=[2])), Frame( send=True, data=Publish(duplicate=False, qos_level=0, topic_name=u"foo", - payload=b"abc", retain=False).serialise()), + payload=b"abc", retain=False)), Frame( send=False, data=Publish(duplicate=False, qos_level=0, topic_name=u"foo", - payload=b"abc", retain=False).serialise()), + payload=b"abc", retain=False)), Frame( send=True, - data=Disconnect().serialise()), + data=Disconnect()), ConnectionLoss(), ] @@ -203,3 +203,104 @@ def test_self_subscribe(host, port): r.run() return Result("self_subscribe", f.success, f.reason, f.client_transcript) + + +def test_qos2_send_wrong_confirm(host, port): + record = [ + Frame( + send=True, + data=Connect(client_id=u"test_wrong_confirm_qos2", + flags=ConnectFlags(clean_session=True))), + Frame( + send=False, + data=ConnACK(session_present=False, return_code=0)), + Frame( + send=True, + data=Subscribe(packet_identifier=1234, + topic_requests=[SubscriptionTopicRequest(u"foo", 2)])), + Frame( + send=False, + data=SubACK(packet_identifier=1234, return_codes=[2])), + Frame( + send=True, + data=Publish(duplicate=False, qos_level=2, topic_name=u"foo", + payload=b"abc", retain=False, packet_identifier=12)), + Frame( + send=False, + data=[ + PubREC(packet_identifier=12), + Publish(duplicate=False, qos_level=2, topic_name=u"foo", + payload=b"abc", retain=False, packet_identifier=1), + PubCOMP(packet_identifier=12)]), + Frame( + send=True, + data=PubREL(packet_identifier=12)), + Frame( + send=True, + data=PubACK(packet_identifier=1)), + Frame( + send=False, + data=b""), + Frame( + send=True, + data=Disconnect()), + ConnectionLoss(), + ] + + r = SelectReactor() + f = ReplayClientFactory(r, record) + e = TCP4ClientEndpoint(r, host, port) + e.connect(f) + r.run() + + return Result("qos2_wrong_confirm", f.success, f.reason, f.client_transcript) + + + +def test_qos1_send_wrong_confirm(host, port): + record = [ + Frame( + send=True, + data=Connect(client_id=u"test_wrong_confirm_qos1", + flags=ConnectFlags(clean_session=True))), + Frame( + send=False, + data=ConnACK(session_present=False, return_code=0)), + Frame( + send=True, + data=Subscribe(packet_identifier=1234, + topic_requests=[SubscriptionTopicRequest(u"foo", 2)])), + Frame( + send=False, + data=SubACK(packet_identifier=1234, return_codes=[2])), + Frame( + send=True, + data=Publish(duplicate=False, qos_level=1, topic_name=u"foo", + payload=b"abc", retain=False, packet_identifier=12)), + Frame( + send=False, + data=[ + PubACK(packet_identifier=12), + Publish(duplicate=False, qos_level=1, topic_name=u"foo", + payload=b"abc", retain=False, packet_identifier=1)]), + # We send a pubrel to the packet_id expecting a puback + Frame( + send=True, + data=PubREL(packet_identifier=1)), + # ..aaaaand we get a pubcomp back (even though mosquitto warns). + Frame( + send=False, + data=PubCOMP(packet_identifier=1)), + Frame( + send=True, + data=Disconnect()), + ConnectionLoss(), + ] + + r = SelectReactor() + f = ReplayClientFactory(r, record) + e = TCP4ClientEndpoint(r, host, port) + e.connect(f) + r.run() + + return Result("qos1_wrong_confirm", f.success, f.reason, f.client_transcript) From 21301cde756ee9fc0930b890c58f4de6426d5920 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 5 Nov 2016 22:41:21 +1100 Subject: [PATCH 33/48] update us to do what mosquitto do --- crossbar/_log_categories.py | 11 +++++---- crossbar/adapter/mqtt/test/interop.py | 2 +- crossbar/adapter/mqtt/tx.py | 35 ++++++++++++--------------- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/crossbar/_log_categories.py b/crossbar/_log_categories.py index 8212e8388..a5e8d26a2 100644 --- a/crossbar/_log_categories.py +++ b/crossbar/_log_categories.py @@ -78,6 +78,11 @@ "MQ301": "Got a PubREC for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", "MQ302": "Got a PubREC for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", "MQ302": "Got a PubCOMP for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", + "MQ303": "Client '{client_id}' sent a PubACK for a non-QoS 1 Publish", + "MQ304": "Client '{client_id}' sent a PubREC for a non-QoS 2 Publish", + "MQ305": "Client '{client_id}' sent a duplicate PubREC", + "MQ306": "Client '{client_id}' sent a PubCOMP for a non-QoS 2 Publish", + "MQ307": "Client '{client_id}' sent a PubCOMP before a PubREC", "MQ303": "Got a non-allowed QoS value in the publish queue, dropping it.", "MQ400": "MQTT client '{client_id}' timed out after recieving no full packets for {seconds}", "MQ401": "Protocol violation from '{client_id}', terminating connection: {error}", @@ -89,11 +94,7 @@ "MQ503": "Error handling a QoS 0 Publish from '{client_id}', dropping connection", "MQ504": "Error handling a QoS 1 Publish from '{client_id}', dropping connection", "MQ505": "Error handling a QoS 2 Publish from '{client_id}', dropping connection", - "MQ506": "Client '{client_id}' sent a PubACK for a non-QoS 1 Publish, dropping connection", - "MQ507": "Client '{client_id}' sent a PubREC for a non-QoS 2 Publish, dropping connection", - "MQ508": "Client '{client_id}' sent a duplicate PubREC, dropping connection", - "MQ509": "Client '{client_id}' sent a PubCOMP for a non-QoS 2 Publish, dropping connection", - "MQ510": "Client '{client_id}' sent a PubCOMP before a PubREC, dropping connection", + } diff --git a/crossbar/adapter/mqtt/test/interop.py b/crossbar/adapter/mqtt/test/interop.py index 20e172bc2..458c70765 100644 --- a/crossbar/adapter/mqtt/test/interop.py +++ b/crossbar/adapter/mqtt/test/interop.py @@ -52,7 +52,7 @@ def run(host, port): fmt_results = [] for r in results: fmt_results.append((r.name, - "True" if r.success else "False", r.reason, r.transcript)) + "True" if r.success else "False", r.reason if r.reason else "", r.transcript)) t = Texttable() t.set_cols_width([20, 10, 80, 60]) diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index 54564b3d9..2b815359d 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -459,10 +459,9 @@ def _handle_events(self, events): if event.packet_identifier in self.session._publishes_awaiting_ack: if not self.session._publishes_awaiting_ack[event.packet_identifier].qos == 1: - self.log.error(log_category="MQ506", - client_id=self.session.client_id) - self.transport.loseConnection() - returnValue(None) + self.log.warn(log_category="MQ303", + client_id=self.session.client_id) + break # MQTT-4.3.2-1: Release the packet ID del self.session._publishes_awaiting_ack[event.packet_identifier] @@ -478,16 +477,14 @@ def _handle_events(self, events): if event.packet_identifier in self.session._publishes_awaiting_ack: if not self.session._publishes_awaiting_ack[event.packet_identifier].qos == 2: - self.log.error(log_category="MQ507", - client_id=self.session.client_id) - self.transport.loseConnection() - returnValue(None) + self.log.warn(log_category="MQ304", + client_id=self.session.client_id) + break if not self.session._publishes_awaiting_ack[event.packet_identifier].stage == 0: - self.log.error(log_category="MQ508", - client_id=self.session.client_id) - self.transport.loseConnection() - returnValue(None) + self.log.warn(log_category="MQ305", + client_id=self.session.client_id) + break self.session._publishes_awaiting_ack[event.packet_identifier].stage = 1 @@ -513,16 +510,14 @@ def _handle_events(self, events): if event.packet_identifier in self.session._publishes_awaiting_ack: if not self.session._publishes_awaiting_ack[event.packet_identifier].qos == 2: - self.log.error(log_category="MQ509", - client_id=self.session.client_id) - self.transport.loseConnection() - returnValue(None) + self.log.warn(log_category="MQ306", + client_id=self.session.client_id) + break if not self.session._publishes_awaiting_ack[event.packet_identifier].stage == 1: - self.log.error(log_category="MQ510", - client_id=self.session.client_id) - self.transport.loseConnection() - returnValue(None) + self.log.warn(log_category="MQ307", + client_id=self.session.client_id) + break # MQTT-4.3.3-1: Release the packet ID del self.session._publishes_awaiting_ack[event.packet_identifier] From d5b3fd6074c005790437a20f247ff0bda4dbb15c Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sat, 5 Nov 2016 22:43:20 +1100 Subject: [PATCH 34/48] header stuff --- crossbar/adapter/mqtt/test/interop.py | 35 +++++++++++++++++++++ crossbar/adapter/mqtt/test/interop_tests.py | 30 ++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/crossbar/adapter/mqtt/test/interop.py b/crossbar/adapter/mqtt/test/interop.py index 458c70765..640917ea7 100644 --- a/crossbar/adapter/mqtt/test/interop.py +++ b/crossbar/adapter/mqtt/test/interop.py @@ -1,3 +1,38 @@ +##################################################################################### +# +# Copyright (C) Tavendo GmbH +# +# Unless a separate license agreement exists between you and Tavendo GmbH (e.g. you +# have purchased a commercial license), the license terms below apply. +# +# Should you enter into a separate license agreement after having received a copy of +# this software, then the terms of such license agreement replace the terms below at +# the time at which such license agreement becomes effective. +# +# In case a separate license agreement ends, and such agreement ends without being +# replaced by another separate license agreement, the license terms below apply +# from the time at which said agreement ends. +# +# LICENSE TERMS +# +# This program is free software: you can redistribute it and/or modify it under the +# terms of the GNU Affero General Public License, version 3, as published by the +# Free Software Foundation. This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Affero General Public License Version 3 for more details. +# +# You should have received a copy of the GNU Affero General Public license along +# with this program. If not, see . +# +##################################################################################### + +""" +Server interop tests, making sure Crossbar's MQTT adapter responds the same as +other MQTT servers. +""" + from __future__ import print_function import click diff --git a/crossbar/adapter/mqtt/test/interop_tests.py b/crossbar/adapter/mqtt/test/interop_tests.py index 72ed74899..b79a2ebaf 100644 --- a/crossbar/adapter/mqtt/test/interop_tests.py +++ b/crossbar/adapter/mqtt/test/interop_tests.py @@ -1,3 +1,32 @@ +##################################################################################### +# +# Copyright (C) Tavendo GmbH +# +# Unless a separate license agreement exists between you and Tavendo GmbH (e.g. you +# have purchased a commercial license), the license terms below apply. +# +# Should you enter into a separate license agreement after having received a copy of +# this software, then the terms of such license agreement replace the terms below at +# the time at which such license agreement becomes effective. +# +# In case a separate license agreement ends, and such agreement ends without being +# replaced by another separate license agreement, the license terms below apply +# from the time at which said agreement ends. +# +# LICENSE TERMS +# +# This program is free software: you can redistribute it and/or modify it under the +# terms of the GNU Affero General Public License, version 3, as published by the +# Free Software Foundation. This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Affero General Public License Version 3 for more details. +# +# You should have received a copy of the GNU Affero General Public license along +# with this program. If not, see . +# +##################################################################################### from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.selectreactor import SelectReactor @@ -9,6 +38,7 @@ Subscribe, SubscriptionTopicRequest, SubACK, Disconnect) + def test_connect(host, port): record = [ Frame( From 5d2f64ec235cf5a94253e49fd7897a1d89a66890 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Mon, 14 Nov 2016 20:36:11 +1100 Subject: [PATCH 35/48] check in another test --- crossbar/adapter/mqtt/test/test_tx.py | 63 +++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index e846f5612..7e4f2ff53 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -1837,5 +1837,68 @@ def test_non_allowed_qos_in_queue_dropped(self): self.assertEqual(t.value(), b'') + def test_does_not_schedule_if_disconnected(self): + """ + If a publish is sent whilst the client is disconnected, it won't be + flushed. + """ + h = BasicHandler() + sessions, r, t, p, cp = make_test_items(h) + + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=False)).serialise() + ) + + for x in iterbytes(data): + p.dataReceived(x) + + # No queued messages + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + + # Disconnect the client + t.connected = False + t.loseConnection() + p.connectionLost(None) + + # WAMP layer calls send_publish, with QoS 0 + p.send_publish(u"hello", 0, b'some bytes') + + # Not queued + self.assertIsNone(p._flush_publishes) + + sessions, r2, t2, p2, cp2 = make_test_items(h, sessions=sessions) + + # We must NOT have a clean session + data = ( + Connect(client_id=u"test123", + flags=ConnectFlags(clean_session=False)).serialise() + ) + + for x in iterbytes(data): + p2.dataReceived(x) + + # The flushing is queued, so we'll have to spin the reactor + self.assertEqual(p2._flush_publishes.args, (True,)) + + r2.advance(0.1) + + # We should have two events; the ConnACK, and the Publish. The ConnACK + # MUST come first. + events = cp2.data_received(t2.value()) + t2.clear() + self.assertEqual(len(events), 2) + self.assertIsInstance(events[0], ConnACK) + self.assertIsInstance(events[1], Publish) + + publish = Publish(duplicate=False, qos_level=0, retain=False, + topic_name=u"hello", payload=b"some bytes") + self.assertEqual(events[1], publish) + + # It is no longer queued + self.assertEqual(len(sessions[u"test123"].queued_messages), 0) + self.assertFalse(t2.disconnecting) + + class OutOfOrderAckTests(TestCase): pass From be5e4499b198bc40c2d606d410482ae95e1244fe Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Mon, 24 Oct 2016 16:13:44 +1100 Subject: [PATCH 36/48] stop things importing the default reactor (which makes CB actually load KQueue on BSD/macOS now) --- crossbar/adapter/rest/callee.py | 9 ++++++--- crossbar/adapter/rest/subscriber.py | 8 ++++++-- crossbar/common/fswatcher.py | 4 ++-- crossbar/common/process.py | 12 +++++++----- crossbar/common/profiler.py | 2 +- crossbar/twisted/resource.py | 1 - crossbar/worker/router.py | 7 +++++-- 7 files changed, 27 insertions(+), 16 deletions(-) diff --git a/crossbar/adapter/rest/callee.py b/crossbar/adapter/rest/callee.py index 0c4b061f1..bd1f9f623 100644 --- a/crossbar/adapter/rest/callee.py +++ b/crossbar/adapter/rest/callee.py @@ -30,8 +30,6 @@ from __future__ import absolute_import -import treq - from six.moves.urllib.parse import urljoin from twisted.internet.defer import inlineCallbacks, returnValue @@ -43,7 +41,12 @@ class RESTCallee(ApplicationSession): def __init__(self, *args, **kwargs): - self._webtransport = kwargs.pop("webTransport", treq) + self._webtransport = kwargs.pop("webTransport") + + if not self._webtransport: + import treq + self._webtransport = treq + super(RESTCallee, self).__init__(*args, **kwargs) @inlineCallbacks diff --git a/crossbar/adapter/rest/subscriber.py b/crossbar/adapter/rest/subscriber.py index 0c3a9efb5..1156ca414 100644 --- a/crossbar/adapter/rest/subscriber.py +++ b/crossbar/adapter/rest/subscriber.py @@ -30,7 +30,6 @@ from __future__ import absolute_import -import treq import json from functools import partial @@ -50,7 +49,12 @@ class MessageForwarder(ApplicationSession): log = make_logger() def __init__(self, *args, **kwargs): - self._webtransport = kwargs.pop("webTransport", treq) + self._webtransport = kwargs.pop("webTransport") + + if not self._webtransport: + import treq + self._webtransport = treq + super(MessageForwarder, self).__init__(*args, **kwargs) @inlineCallbacks diff --git a/crossbar/common/fswatcher.py b/crossbar/common/fswatcher.py index 680fe1d54..7d9dae2a4 100644 --- a/crossbar/common/fswatcher.py +++ b/crossbar/common/fswatcher.py @@ -33,8 +33,6 @@ import os from txaio import make_logger -from twisted.internet import reactor - try: from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler @@ -83,6 +81,8 @@ def on_any_event(evt): u'rel_path': os.path.relpath(evt.src_path, self._working_dir), u'is_directory': evt.is_directory, } + + from twisted.internet import reactor reactor.callFromThread(callback, event) self._handler.on_any_event = on_any_event diff --git a/crossbar/common/process.py b/crossbar/common/process.py index e45d47326..13fd4d41f 100644 --- a/crossbar/common/process.py +++ b/crossbar/common/process.py @@ -44,11 +44,7 @@ # # twisted.conch.manhole_ssh will import even without, but we _need_ SSH import pyasn1 # noqa - from twisted.cred import checkers, portal - from twisted.conch.manhole import ColoredManhole - from twisted.conch.manhole_ssh import ConchFactory, \ - TerminalRealm, \ - TerminalSession + import cryptography # noqa except ImportError as e: _HAS_MANHOLE = False _MANHOLE_MISSING_REASON = str(e) @@ -64,6 +60,8 @@ from txaio import make_logger +from twisted.cred import checkers, portal + from crossbar.common import checkconfig from crossbar.common.checkconfig import get_config_value from crossbar.twisted.endpoint import create_listening_port_from_config @@ -576,6 +574,10 @@ class PatchedTerminalSession(TerminalSession): def windowChanged(self, winSize): pass + from twisted.conch.manhole_ssh import ( + ConchFactory, TerminalRealm, TerminalSession) + from twisted.conch.manhole import ColoredManhole + rlm = TerminalRealm() rlm.sessionFactory = PatchedTerminalSession # monkey patch rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(namespace) diff --git a/crossbar/common/profiler.py b/crossbar/common/profiler.py index 8e6d8ff82..96f332d32 100644 --- a/crossbar/common/profiler.py +++ b/crossbar/common/profiler.py @@ -35,7 +35,6 @@ import six -from twisted.internet import reactor from twisted.internet.defer import Deferred from twisted.internet.threads import deferToThread @@ -221,6 +220,7 @@ def cleanup(res): self.log.info("Starting profiling using {profiler} for {runtime} seconds.", profiler=self._id, runtime=runtime) + from twisted.internet import reactor reactor.callLater(runtime, finish_profile) return self._profile_id, self._finished diff --git a/crossbar/twisted/resource.py b/crossbar/twisted/resource.py index 6ac2dbfa5..a8f8ff100 100644 --- a/crossbar/twisted/resource.py +++ b/crossbar/twisted/resource.py @@ -37,7 +37,6 @@ from twisted.web.http import NOT_FOUND from twisted.web.resource import Resource, NoResource from twisted.web.static import File -from twisted.web.proxy import ReverseProxyResource # noqa (Republish resource) from twisted.python.filepath import FilePath from txaio import make_logger diff --git a/crossbar/worker/router.py b/crossbar/worker/router.py index 3a22c3bc9..05352cccb 100644 --- a/crossbar/worker/router.py +++ b/crossbar/worker/router.py @@ -90,8 +90,7 @@ from crossbar.twisted.resource import JsonResource, \ Resource404, \ - RedirectResource, \ - ReverseProxyResource + RedirectResource from crossbar.adapter.mqtt.wamp import WampMQTTServerFactory @@ -1062,6 +1061,10 @@ def _create_resource(self, path_config, nested=True): # Reverse proxy resource # elif path_config['type'] == 'reverseproxy': + + # Import late because t.w.proxy imports the reactor + from twisted.web.proxy import ReverseProxyResource + host = path_config['host'] port = int(path_config.get('port', 80)) path = path_config.get('path', '').encode('ascii', 'ignore') From 2278d538e1c282cd33d216e74a693ed7115399f4 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Mon, 21 Nov 2016 00:16:01 +1100 Subject: [PATCH 37/48] remove ack handling from the adapter, delegate it to the wamp layer (which will be delegated to the router when QoS1 and resending messages are implemented in WAMP) --- crossbar/adapter/mqtt/test/test_tx.py | 160 +++----------------------- crossbar/adapter/mqtt/tx.py | 130 ++++++--------------- crossbar/adapter/mqtt/wamp.py | 12 ++ 3 files changed, 63 insertions(+), 239 deletions(-) diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index 29d214a9a..af198fd94 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -35,7 +35,7 @@ from binascii import unhexlify from crossbar.adapter.mqtt.tx import ( - MQTTServerTwistedProtocol, AwaitingACK, Session, Message) + MQTTServerTwistedProtocol, Session, Message) from crossbar.adapter.mqtt.protocol import ( MQTTParser, MQTTClientParser, client_packet_handlers, P_CONNACK) from crossbar.adapter.mqtt._events import ( @@ -68,6 +68,18 @@ def new_wamp_session(self, event): def existing_wamp_session(self, event): return None + def process_puback(self, event): + return + + def process_pubrec(self, event): + return + + def process_pubrel(self, event): + return + + def process_pubcomp(self, event): + return + def make_test_items(handler, sessions=None): @@ -1246,12 +1258,6 @@ def test_qos_1_queues_message(self): self.assertEqual(len(events), 1) self.assertEqual(events[0], expected_publish) - # Server is still awaiting the client's response - self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) - self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], - AwaitingACK(qos=1, stage=0, message=expected_publish)) - # We send the PubACK, which we don't get a response to puback = PubACK(packet_identifier=4567) @@ -1261,54 +1267,8 @@ def test_qos_1_queues_message(self): events = cp.data_received(t.value()) self.assertEqual(len(events), 0) - # It is no longer queued - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 0) - self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) self.assertFalse(t.disconnecting) - def test_qos_1_unassociated_puback_warns(self): - """ - On receiving a PubACK which does not reference any known existing - Publish the server has sent, it will drop it and raise a warning. - - XXX: Spec is unclear if this is the proper behaviour! - """ - got_packets = [] - - h = BasicHandler() - sessions, r, t, p, cp = make_test_items(h) - - data = ( - Connect(client_id=u"test123", - flags=ConnectFlags(clean_session=True)).serialise() - ) - - for x in iterbytes(data): - p.dataReceived(x) - - # Connect has happened - events = cp.data_received(t.value()) - t.clear() - self.assertFalse(t.disconnecting) - self.assertIsInstance(events[0], ConnACK) - - # We send the PubACK, which we don't get a response to (and also - # doesn't refer to any Publish the client was sent) - puback = PubACK(packet_identifier=4567) - - with LogCapturer("trace") as logs: - for x in iterbytes(puback.serialise()): - p.dataReceived(x) - - events = cp.data_received(t.value()) - self.assertEqual(len(events), 0) - - # A warning is raised - sent_logs = logs.get_category("MQ300") - self.assertEqual(len(sent_logs), 1) - self.assertEqual(sent_logs[0]["log_level"], LogLevel.warn) - self.assertEqual(sent_logs[0]["pub_id"], 4567) - def test_qos_2_queues_message(self): """ The WAMP layer calling send_publish will queue a message up for @@ -1356,12 +1316,6 @@ def test_qos_2_queues_message(self): self.assertEqual(len(events), 1) self.assertEqual(events[0], expected_publish) - # Server is still awaiting the client's response - self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) - self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], - AwaitingACK(qos=2, stage=0, message=expected_publish)) - # We send the PubREC, which we should get a PubREL back with pubrec = PubREC(packet_identifier=4567) @@ -1373,21 +1327,12 @@ def test_qos_2_queues_message(self): self.assertEqual(len(events), 1) self.assertEqual(events[0], PubREL(packet_identifier=4567)) - # Server is still awaiting the client's response, it is not complete - self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) - self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], - AwaitingACK(qos=2, stage=1, message=expected_publish)) - # We send the PubCOMP, which has no response pubcomp = PubCOMP(packet_identifier=4567) for x in iterbytes(pubcomp.serialise()): p.dataReceived(x) - # It is no longer queued - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 0) - self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) self.assertFalse(t.disconnecting) def test_qos_1_resent_on_disconnect(self): @@ -1433,13 +1378,6 @@ def test_qos_1_resent_on_disconnect(self): self.assertEqual(len(events), 2) self.assertEqual(events[1], expected_publish) - # Server is still awaiting the client's response, message is not queued - self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) - self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], - AwaitingACK(qos=1, stage=0, message=expected_publish)) - self.assertEqual(len(sessions[u"test123"].queued_messages), 0) - # Disconnect the client t.connected = False t.loseConnection() @@ -1457,8 +1395,6 @@ def test_qos_1_resent_on_disconnect(self): p2.dataReceived(x) # The flushing is queued, so we'll have to spin the reactor - self.assertEqual(p2._flush_publishes.args, (True,)) - r2.advance(0.1) # We should have two events; the ConnACK, and the Publish. The ConnACK @@ -1484,10 +1420,6 @@ def test_qos_1_resent_on_disconnect(self): events = cp2.data_received(t2.value()) self.assertEqual(len(events), 0) - # It is no longer queued - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 0) - self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) - self.assertEqual(len(sessions[u"test123"].queued_messages), 0) self.assertFalse(t2.disconnecting) def test_qos_2_resent_on_disconnect_pubrel(self): @@ -1533,13 +1465,6 @@ def test_qos_2_resent_on_disconnect_pubrel(self): self.assertEqual(len(events), 2) self.assertEqual(events[1], expected_publish) - # Server is still awaiting the client's response, message is not queued - self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) - self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], - AwaitingACK(qos=2, stage=0, message=expected_publish)) - self.assertEqual(len(sessions[u"test123"].queued_messages), 0) - # Disconnect the client t.connected = False t.loseConnection() @@ -1557,8 +1482,6 @@ def test_qos_2_resent_on_disconnect_pubrel(self): p2.dataReceived(x) # The flushing is queued, so we'll have to spin the reactor - self.assertEqual(p2._flush_publishes.args, (True,)) - r2.advance(0.1) # We should have two events; the ConnACK, and the Publish. The ConnACK @@ -1575,14 +1498,6 @@ def test_qos_2_resent_on_disconnect_pubrel(self): payload=b"some bytes") self.assertEqual(events[1], resent_publish) - # Server is still waiting for us to send the PubREC -- QoS2, Stage 0 - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) - self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) - self.assertEqual(len(sessions[u"test123"].queued_messages), 0) - self.assertFalse(t2.disconnecting) - self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], - AwaitingACK(qos=2, stage=0, message=resent_publish)) - # We send the PubREC to this Publish pubrec = PubREC(packet_identifier=4567) @@ -1595,14 +1510,6 @@ def test_qos_2_resent_on_disconnect_pubrel(self): self.assertEqual(len(events), 1) self.assertEqual(events[0], PubREL(packet_identifier=4567)) - # Server is still waiting for us to send the PubCOMP -- QoS2, Stage 1 - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) - self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) - self.assertEqual(len(sessions[u"test123"].queued_messages), 0) - self.assertFalse(t2.disconnecting) - self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], - AwaitingACK(qos=2, stage=1, message=resent_publish)) - # We send the PubCOMP to this Publish pubcomp = PubCOMP(packet_identifier=4567) @@ -1613,10 +1520,6 @@ def test_qos_2_resent_on_disconnect_pubrel(self): events = cp2.data_received(t2.value()) self.assertEqual(len(events), 0) - # Not queued, not awaiting ACK - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 0) - self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) - self.assertEqual(len(sessions[u"test123"].queued_messages), 0) self.assertFalse(t2.disconnecting) def test_qos_2_resent_on_disconnect_pubcomp(self): @@ -1662,14 +1565,6 @@ def test_qos_2_resent_on_disconnect_pubcomp(self): self.assertEqual(len(events), 2) self.assertEqual(events[1], expected_publish) - # Server is waiting for us to send the PubREC -- QoS2, Stage 0 - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) - self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) - self.assertEqual(len(sessions[u"test123"].queued_messages), 0) - self.assertFalse(t.disconnecting) - self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], - AwaitingACK(qos=2, stage=0, message=expected_publish)) - # We send the PubREC to this Publish pubrec = PubREC(packet_identifier=4567) @@ -1682,14 +1577,6 @@ def test_qos_2_resent_on_disconnect_pubcomp(self): self.assertEqual(len(events), 1) self.assertEqual(events[0], PubREL(packet_identifier=4567)) - # Server is waiting for a PubCOMP -- QoS2, Stage 1 - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) - self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) - self.assertEqual(len(sessions[u"test123"].queued_messages), 0) - self.assertFalse(t.disconnecting) - self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], - AwaitingACK(qos=2, stage=1, message=expected_publish)) - # Disconnect the client t.connected = False t.loseConnection() @@ -1707,7 +1594,6 @@ def test_qos_2_resent_on_disconnect_pubcomp(self): p2.dataReceived(x) # The flushing is queued, so we'll have to spin the reactor - self.assertEqual(p2._flush_publishes.args, (True,)) r2.advance(0.1) # Should get a resent PubREL back @@ -1717,13 +1603,7 @@ def test_qos_2_resent_on_disconnect_pubcomp(self): self.assertIsInstance(events[0], ConnACK) self.assertEqual(events[1], PubREL(packet_identifier=4567)) - # Server is still waiting for us to send the PubCOMP -- QoS2, Stage 1 - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 1) - self.assertIn(4567, sessions[u"test123"]._in_flight_packet_ids) - self.assertEqual(len(sessions[u"test123"].queued_messages), 0) self.assertFalse(t2.disconnecting) - self.assertEqual(sessions[u"test123"]._publishes_awaiting_ack[4567], - AwaitingACK(qos=2, stage=1, message=expected_publish)) # We send the PubCOMP to this PubREL pubcomp = PubCOMP(packet_identifier=4567) @@ -1735,10 +1615,6 @@ def test_qos_2_resent_on_disconnect_pubcomp(self): events = cp2.data_received(t2.value()) self.assertEqual(len(events), 0) - # Not queued, not awaiting ACK - self.assertEqual(len(sessions[u"test123"]._publishes_awaiting_ack), 0) - self.assertNotIn(4567, sessions[u"test123"]._in_flight_packet_ids) - self.assertEqual(len(sessions[u"test123"].queued_messages), 0) self.assertFalse(t2.disconnecting) def test_non_allowed_qos_not_queued(self): @@ -1870,8 +1746,6 @@ def test_does_not_schedule_if_disconnected(self): p2.dataReceived(x) # The flushing is queued, so we'll have to spin the reactor - self.assertEqual(p2._flush_publishes.args, (True,)) - r2.advance(0.1) # We should have two events; the ConnACK, and the Publish. The ConnACK @@ -1890,6 +1764,8 @@ def test_does_not_schedule_if_disconnected(self): self.assertEqual(len(sessions[u"test123"].queued_messages), 0) self.assertFalse(t2.disconnecting) - -class OutOfOrderAckTests(TestCase): - pass + for x in [test_qos_1_resent_on_disconnect, + test_qos_2_resent_on_disconnect_pubcomp, + test_qos_2_resent_on_disconnect_pubrel]: + x.todo = ("Needs WAMP-level implementation first, and the WAMP router " + "to resend ACKs/messages") diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index d4eb27f3d..95f4de8f5 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -66,12 +66,10 @@ class Session(object): subscriptions = attr.ib(default=attr.Factory(dict)) connected = attr.ib(default=False) survives = attr.ib(default=False) - _in_flight_packet_ids = attr.ib(default=attr.Factory(set)) - _publishes_awaiting_ack = attr.ib(default=attr.Factory(collections.OrderedDict)) def get_packet_id(self): x = 0 - while x == 0 or x in self._in_flight_packet_ids: + while x == 0: x = randint(1, _SIXTEEN_BIT_MAX) return x @@ -84,14 +82,6 @@ class Message(object): qos = attr.ib() -@attr.s -class AwaitingACK(object): - - qos = attr.ib() - stage = attr.ib() - message = attr.ib() - - class MQTTServerTwistedProtocol(Protocol): log = make_logger() @@ -165,10 +155,6 @@ def _send_publish(self, topic, qos, body): packet_identifier=packet_id, topic_name=topic, payload=body) - waiting_ack = AwaitingACK(qos=1, stage=0, message=publish) - self.session._publishes_awaiting_ack[packet_id] = waiting_ack - self.session._in_flight_packet_ids.add(packet_id) - elif qos == 2: packet_id = self.session.get_packet_id() @@ -176,10 +162,6 @@ def _send_publish(self, topic, qos, body): packet_identifier=packet_id, topic_name=topic, payload=body) - waiting_ack = AwaitingACK(qos=2, stage=0, message=publish) - self.session._publishes_awaiting_ack[packet_id] = waiting_ack - self.session._in_flight_packet_ids.add(packet_id) - else: self.log.warn(log_category="MQ303") return @@ -197,7 +179,7 @@ def _send_packet(self, packet): packet=packet, conn_id=self._connection_id) self.transport.write(packet.serialise()) - def _flush_saved_messages(self, including_non_acked=False): + def _flush_saved_messages(self): if self._flush_publishes: self._flush_publishes = None @@ -206,28 +188,6 @@ def _flush_saved_messages(self, including_non_acked=False): if not self.transport.connected: return None - if including_non_acked: - for message in self.session._publishes_awaiting_ack.values(): - if message.qos == 1: - message.message.duplicate = True - self._send_packet(message.message) - if message.qos == 2: - if message.stage == 0: - # Stage 0 == Publish sent - # Resend Publish - message.message.duplicate = True - self._send_packet(message.message) - - elif message.stage == 1: - # Stage 1 == PubREC got, PubREL sent - # Resend PubREL - pkt = PubREL(packet_identifier=message.message.packet_identifier) - self._send_packet(pkt) - - # Invalid! - else: - pass - # New, queued messages while self.session.queued_messages: message = self.session.queued_messages.popleft() @@ -301,7 +261,7 @@ def _handle_events(self, events): if event.client_id in self._mqtt_sessions: self.session = self._mqtt_sessions[event.client_id] self._flush_publishes = self._reactor.callLater( - 0, self._flush_saved_messages, True) + 0, self._flush_saved_messages) self._handler.existing_wamp_session(self.session) # Have a session, set to 1/True as in MQTT-3.2.2-2 session_present = True @@ -456,42 +416,23 @@ def _handle_events(self, events): elif isinstance(event, PubACK): - if event.packet_identifier in self.session._publishes_awaiting_ack: - - if not self.session._publishes_awaiting_ack[event.packet_identifier].qos == 1: - self.log.warn(log_category="MQ303", - client_id=self.session.client_id) - break - - # MQTT-4.3.2-1: Release the packet ID - del self.session._publishes_awaiting_ack[event.packet_identifier] - self.session._in_flight_packet_ids.remove(event.packet_identifier) - - else: - self.log.warn( - log_category="MQ300", client_id=self.session.client_id, - pub_id=event.packet_identifier) + try: + self._handler.process_puback(event) + except: + # MQTT-4.8.0-2 - If we get a transient errorm we must close + # the connection. + self.transport.loseConnection() + returnValue(None) elif isinstance(event, PubREC): - if event.packet_identifier in self.session._publishes_awaiting_ack: - - if not self.session._publishes_awaiting_ack[event.packet_identifier].qos == 2: - self.log.warn(log_category="MQ304", - client_id=self.session.client_id) - break - - if not self.session._publishes_awaiting_ack[event.packet_identifier].stage == 0: - self.log.warn(log_category="MQ305", - client_id=self.session.client_id) - break - - self.session._publishes_awaiting_ack[event.packet_identifier].stage = 1 - - else: - self.log.warn( - log_category="MQ301", client_id=self.session.client_id, - pub_id=event.packet_identifier) + try: + self._handler.process_pubrec(event) + except: + # MQTT-4.8.0-2 - If we get a transient errorm we must close + # the connection. + self.transport.loseConnection() + returnValue(None) # MQTT-4.3.3-1: MUST send back a PubREL -- even if it's not an # ID we know about, apparently, according to Mosquitto and @@ -500,33 +441,28 @@ def _handle_events(self, events): self._send_packet(resp) elif isinstance(event, PubREL): - # Should check if it is valid here + + try: + self._handler.process_pubrel(event) + except: + # MQTT-4.8.0-2 - If we get a transient error we must close + # the connection. + self.transport.loseConnection() + returnValue(None) + resp = PubCOMP(packet_identifier=event.packet_identifier) self._send_packet(resp) continue elif isinstance(event, PubCOMP): - if event.packet_identifier in self.session._publishes_awaiting_ack: - - if not self.session._publishes_awaiting_ack[event.packet_identifier].qos == 2: - self.log.warn(log_category="MQ306", - client_id=self.session.client_id) - break - - if not self.session._publishes_awaiting_ack[event.packet_identifier].stage == 1: - self.log.warn(log_category="MQ307", - client_id=self.session.client_id) - break - - # MQTT-4.3.3-1: Release the packet ID - del self.session._publishes_awaiting_ack[event.packet_identifier] - self.session._in_flight_packet_ids.remove(event.packet_identifier) - - else: - self.log.warn( - log_category="MQ302", client_id=self.session.client_id, - pub_id=event.packet_identifier) + try: + self._handler.process_pubcomp(event) + except: + # MQTT-4.8.0-2 - If we get a transient error we must close + # the connection. + self.transport.loseConnection() + returnValue(None) elif isinstance(event, Disconnect): # TODO: get rid of some will messages diff --git a/crossbar/adapter/mqtt/wamp.py b/crossbar/adapter/mqtt/wamp.py index 6d636261a..d635fd344 100644 --- a/crossbar/adapter/mqtt/wamp.py +++ b/crossbar/adapter/mqtt/wamp.py @@ -85,6 +85,18 @@ def process_publish_qos_1(self, event): return self._publish(event, options=PublishOptions(acknowledge=True, exclude_me=False)) + def process_puback(self, event): + return + + def process_pubrec(self, event): + return + + def process_pubrel(self, event): + return + + def process_pubcomp(self, event): + return + @inlineCallbacks def process_subscribe(self, packet): From 18fbd6397f2dd68e61ba7787c1c7e471dddf15dd Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Mon, 21 Nov 2016 00:20:36 +1100 Subject: [PATCH 38/48] update license file here too --- crossbar/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crossbar/__main__.py b/crossbar/__main__.py index a253cc467..3244d3382 100644 --- a/crossbar/__main__.py +++ b/crossbar/__main__.py @@ -1,9 +1,9 @@ ##################################################################################### # -# Copyright (C) Tavendo GmbH +# Copyright (c) Crossbar.io Technologies GmbH # -# Unless a separate license agreement exists between you and Tavendo GmbH (e.g. you -# have purchased a commercial license), the license terms below apply. +# Unless a separate license agreement exists between you and Crossbar.io GmbH (e.g. +# you have purchased a commercial license), the license terms below apply. # # Should you enter into a separate license agreement after having received a copy of # this software, then the terms of such license agreement replace the terms below at From 1b8d562387cf262083a74890badb1236b88af694 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Mon, 24 Oct 2016 16:14:11 +1100 Subject: [PATCH 39/48] use plistlib to get the serial number of a macOS machine, fixes #870 --- crossbar/controller/node.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/crossbar/controller/node.py b/crossbar/controller/node.py index b4e956d7f..1705cdfb9 100644 --- a/crossbar/controller/node.py +++ b/crossbar/controller/node.py @@ -36,6 +36,8 @@ import getpass import pkg_resources import binascii +import six +import subprocess from collections import OrderedDict import pyqrcode @@ -46,6 +48,7 @@ import twisted from twisted.internet.defer import inlineCallbacks, Deferred from twisted.internet.ssl import optionsForClientTLS +from twisted.python.runtime import platform from txaio import make_logger @@ -162,12 +165,29 @@ def _machine_id(): """ for informational purposes, try to get a machine unique id thing """ - try: - # why this? see: http://0pointer.de/blog/projects/ids.html - with open('/var/lib/dbus/machine-id', 'r') as f: - return f.read().strip() - except: - # OS X? Something else? Get a hostname, at least. + if platform.isLinux(): + try: + # why this? see: http://0pointer.de/blog/projects/ids.html + with open('/var/lib/dbus/machine-id', 'r') as f: + return f.read().strip() + except: + # Non-dbus using Linux, get a hostname + return socket.gethostname() + + elif platform.isMacOSX(): + # Get the serial number of the platform + import plistlib + plist_data = subprocess.check_output(["ioreg", "-rd1", "-c", "IOPlatformExpertDevice", "-a"]) + + if six.PY2: + # Only API on 2.7 + return plistlib.readPlistFromString(plist_data)["IOPlatformSerialNumber"] + else: + # New, non-deprecated 3.4+ API + return plistlib.loads(plist_data)[0]["IOPlatformSerialNumber"] + + else: + # Something else, just get a hostname return socket.gethostname() From 49f3f68b4c917f2492c6f771b95d232fe896ba90 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Mon, 21 Nov 2016 12:15:17 +1100 Subject: [PATCH 40/48] fix #873, cookie name showing up wrong in logs --- crossbar/newsfragments/873.bugfix | 1 + crossbar/router/protocol.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 crossbar/newsfragments/873.bugfix diff --git a/crossbar/newsfragments/873.bugfix b/crossbar/newsfragments/873.bugfix new file mode 100644 index 000000000..12d75ea5e --- /dev/null +++ b/crossbar/newsfragments/873.bugfix @@ -0,0 +1 @@ +When using cookie authentication, the debug logs will now mention the correct cookie name if it is not the default. diff --git a/crossbar/router/protocol.py b/crossbar/router/protocol.py index 2c5cfbd97..ded41e45d 100644 --- a/crossbar/router/protocol.py +++ b/crossbar/router/protocol.py @@ -243,8 +243,8 @@ def print_traffic(): # there is a cookie set, and the cookie was previously successfully authenticated, # so immediately authenticate the client using that information self._authprovider = u'cookie' - self.log.debug("Authenticated client via cookie cbtid={cbtid} as authid={authid}, authrole={authrole}, authmethod={authmethod}, authrealm={authrealm}", - cbtid=self._cbtid, authid=self._authid, authrole=self._authrole, authmethod=self._authmethod, authrealm=self._authrealm) + self.log.debug("Authenticated client via cookie {cookiename}={cbtid} as authid={authid}, authrole={authrole}, authmethod={authmethod}, authrealm={authrealm}", + cookiename=self.factory._cookiestore._cookie_id_field, cbtid=self._cbtid, authid=self._authid, authrole=self._authrole, authmethod=self._authmethod, authrealm=self._authrealm) else: # there is a cookie set, but the cookie wasn't authenticated yet using a different auth method self.log.debug("Cookie-based authentication enabled, but cookie isn't authenticated yet") From 200042e47e7a402287695ffe3dbcccd1e84a9a9a Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Mon, 21 Nov 2016 12:16:35 +1100 Subject: [PATCH 41/48] add authextra to the block --- crossbar/newsfragments/895.bugfix | 1 + crossbar/router/cookiestore.py | 24 ++++++++++++++---------- crossbar/router/session.py | 4 ++-- 3 files changed, 17 insertions(+), 12 deletions(-) create mode 100644 crossbar/newsfragments/895.bugfix diff --git a/crossbar/newsfragments/895.bugfix b/crossbar/newsfragments/895.bugfix new file mode 100644 index 000000000..d17f4a044 --- /dev/null +++ b/crossbar/newsfragments/895.bugfix @@ -0,0 +1 @@ +Crossbar's cookie authentication now sends the `authextra` block to the client. diff --git a/crossbar/router/cookiestore.py b/crossbar/router/cookiestore.py index f1baca04d..96a05cc5b 100644 --- a/crossbar/router/cookiestore.py +++ b/crossbar/router/cookiestore.py @@ -120,6 +120,7 @@ def create(self): 'authrole': None, 'authrealm': None, 'authmethod': None, + 'authextra': None, # set of WAMP transports (WebSocket connections) this # cookie is currently used on @@ -145,21 +146,21 @@ def exists(self, cbtid): def getAuth(self, cbtid): """ - Return `(authid, authrole, authmethod, authrealm)` tuple given cookie ID. + Return `(authid, authrole, authmethod, authrealm, authextra)` tuple given cookie ID. """ if cbtid in self._cookies: c = self._cookies[cbtid] - cookie_auth_info = c['authid'], c['authrole'], c['authmethod'], c['authrealm'] + cookie_auth_info = c['authid'], c['authrole'], c['authmethod'], c['authrealm'], c['authextra'] else: - cookie_auth_info = None, None, None, None + cookie_auth_info = None, None, None, None, None self.log.debug("Cookie auth info for {cbtid} retrieved: {cookie_auth_info}", cbtid=cbtid, cookie_auth_info=cookie_auth_info) return cookie_auth_info - def setAuth(self, cbtid, authid, authrole, authmethod, authrealm): + def setAuth(self, cbtid, authid, authrole, authmethod, authextra, authrealm): """ - Set `(authid, authrole, authmethod)` triple for given cookie ID. + Set `(authid, authrole, authmethod, authextra)` for given cookie ID. """ if cbtid in self._cookies: c = self._cookies[cbtid] @@ -167,6 +168,7 @@ def setAuth(self, cbtid, authid, authrole, authmethod, authrealm): c['authrole'] = authrole c['authrealm'] = authrealm c['authmethod'] = authmethod + c['authextra'] = authextra def addProto(self, cbtid, proto): """ @@ -262,7 +264,8 @@ def _persist(self, id, c, status='created'): 'id': id, status: c['created'], 'max_age': c['max_age'], 'authid': c['authid'], 'authrole': c['authrole'], 'authmethod': c['authmethod'], - 'authrealm': c['authrealm'] + 'authrealm': c['authrealm'], + 'authextra': c['authextra'], }) + '\n') self._cookie_file.flush() os.fsync(self._cookie_file.fileno()) @@ -289,15 +292,15 @@ def create(self): return cbtid, header - def setAuth(self, cbtid, authid, authrole, authmethod, authrealm): + def setAuth(self, cbtid, authid, authrole, authmethod, authextra, authrealm): if self.exists(cbtid): cookie = self._cookies[cbtid] # only set the changes and write them to the file if any of the values changed - if authid != cookie['authid'] or authrole != cookie['authrole'] or authmethod != cookie['authmethod'] or authrealm != cookie['authrealm']: - CookieStore.setAuth(self, cbtid, authid, authrole, authmethod, authrealm) + if authid != cookie['authid'] or authrole != cookie['authrole'] or authmethod != cookie['authmethod'] or authrealm != cookie['authrealm'] or authextra != cookie['authextra']: + CookieStore.setAuth(self, cbtid, authid, authrole, authmethod, authextra, authrealm) self._persist(cbtid, cookie, status='modified') def _clean_cookie_file(self): @@ -316,7 +319,8 @@ def _clean_cookie_file(self): 'authid': cookie['authid'], 'authrole': cookie['authrole'], 'authmethod': cookie['authmethod'], - 'authrealm': cookie['authrealm'] + 'authrealm': cookie['authrealm'], + 'authextra': cookie['authextra'], }) + '\n' cookie_file.write(cookie_record) diff --git a/crossbar/router/session.py b/crossbar/router/session.py index aa1bbe53a..8870d44c2 100644 --- a/crossbar/router/session.py +++ b/crossbar/router/session.py @@ -735,7 +735,7 @@ def onJoin(self, details): if hasattr(self._transport, '_cbtid') and self._transport._cbtid: if details.authmethod != 'cookie': - self._transport.factory._cookiestore.setAuth(self._transport._cbtid, details.authid, details.authrole, details.authmethod, self._realm) + self._transport.factory._cookiestore.setAuth(self._transport._cbtid, details.authid, details.authrole, details.authmethod, details.authextra, self._realm) # Router/Realm service session # @@ -780,7 +780,7 @@ def onLeave(self, details): cs = self._transport.factory._cookiestore # set cookie to "not authenticated" - cs.setAuth(self._transport._cbtid, None, None, None, None) + cs.setAuth(self._transport._cbtid, None, None, None, None, None) # kick all session for the same auth cookie for proto in cs.getProtos(self._transport._cbtid): From 074fc287c1ea71e0b510765f08c41b44f5bd7e0a Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Mon, 21 Nov 2016 12:45:15 +1100 Subject: [PATCH 42/48] amend tests --- crossbar/router/test/test_cookiestore.py | 36 ++++++++++++++++-------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/crossbar/router/test/test_cookiestore.py b/crossbar/router/test/test_cookiestore.py index ce8ad4439..fac4485b2 100644 --- a/crossbar/router/test/test_cookiestore.py +++ b/crossbar/router/test/test_cookiestore.py @@ -32,7 +32,8 @@ def test_purge_on_startup(self): "authid": "example.authid", "authrole": "example.authrole", "authrealm": "example.authrealm", - "authmethod": "example.authmethod" + "authmethod": "example.authmethod", + "authextra": {}, }, { "id": "thisIsAnotherID", @@ -41,7 +42,8 @@ def test_purge_on_startup(self): "authid": "example.other.authid", "authrole": "example.other.authrole", "authrealm": "example.other.authrealm", - "authmethod": "example.other.authmethod" + "authmethod": "example.other.authmethod", + "authextra": {"a": "b"}, }, { "id": "thisIsAnID", @@ -50,7 +52,8 @@ def test_purge_on_startup(self): "authid": "example.second.authid", "authrole": "example.second.authrole", "authrealm": "example.second.authrealm", - "authmethod": "example.second.authmethod" + "authmethod": "example.second.authmethod", + "authextra": {}, } ] @@ -62,7 +65,8 @@ def test_purge_on_startup(self): "authid": "example.second.authid", "authrole": "example.second.authrole", "authrealm": "example.second.authrealm", - "authmethod": "example.second.authmethod" + "authmethod": "example.second.authmethod", + "authextra": {}, }, { "id": "thisIsAnotherID", @@ -71,7 +75,8 @@ def test_purge_on_startup(self): "authid": "example.other.authid", "authrole": "example.other.authrole", "authrealm": "example.other.authrealm", - "authmethod": "example.other.authmethod" + "authmethod": "example.other.authmethod", + "authextra": {"a": "b"}, } ] @@ -101,7 +106,8 @@ def test_purge_on_startup_default_is_false(self): "authid": "example.authid", "authrole": "example.authrole", "authrealm": "example.authrealm", - "authmethod": "example.authmethod" + "authmethod": "example.authmethod", + "authextra": {}, }, { "id": "thisIsAnID", @@ -110,7 +116,8 @@ def test_purge_on_startup_default_is_false(self): "authid": "example.second.authid", "authrole": "example.second.authrole", "authrealm": "example.second.authrealm", - "authmethod": "example.second.authmethod" + "authmethod": "example.second.authmethod", + "authextra": {}, }, { "id": "thisIsAnotherID", @@ -119,7 +126,8 @@ def test_purge_on_startup_default_is_false(self): "authid": "example.other.authid", "authrole": "example.other.authrole", "authrealm": "example.other.authrealm", - "authmethod": "example.other.authmethod" + "authmethod": "example.other.authmethod", + "authextra": {}, } ] @@ -153,7 +161,8 @@ def test_purge_on_startup_delete_expired_cookies(self): "authid": "example.authid", "authrole": "example.authrole", "authrealm": "example.authrealm", - "authmethod": "example.authmethod" + "authmethod": "example.authmethod", + "authextra": {}, }, { "id": "thisIsAnotherID", @@ -162,7 +171,8 @@ def test_purge_on_startup_delete_expired_cookies(self): "authid": "example.other.authid", "authrole": "example.other.authrole", "authrealm": "example.other.authrealm", - "authmethod": "example.other.authmethod" + "authmethod": "example.other.authmethod", + "authextra": {}, }, { "id": "thisIsAnID", @@ -171,7 +181,8 @@ def test_purge_on_startup_delete_expired_cookies(self): "authid": "example.second.authid", "authrole": "example.second.authrole", "authrealm": "example.second.authrealm", - "authmethod": "example.second.authmethod" + "authmethod": "example.second.authmethod", + "authextra": {}, } ] @@ -183,7 +194,8 @@ def test_purge_on_startup_delete_expired_cookies(self): "authid": "example.other.authid", "authrole": "example.other.authrole", "authrealm": "example.other.authrealm", - "authmethod": "example.other.authmethod" + "authmethod": "example.other.authmethod", + "authextra": {}, }, ] From ca628165a0a7be0c36205010b16f244deaaccb02 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Sun, 30 Oct 2016 16:50:25 +1100 Subject: [PATCH 43/48] make python -m crossbar work, using the same console script --- crossbar/__main__.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 crossbar/__main__.py diff --git a/crossbar/__main__.py b/crossbar/__main__.py new file mode 100644 index 000000000..a253cc467 --- /dev/null +++ b/crossbar/__main__.py @@ -0,0 +1,37 @@ +##################################################################################### +# +# Copyright (C) Tavendo GmbH +# +# Unless a separate license agreement exists between you and Tavendo GmbH (e.g. you +# have purchased a commercial license), the license terms below apply. +# +# Should you enter into a separate license agreement after having received a copy of +# this software, then the terms of such license agreement replace the terms below at +# the time at which such license agreement becomes effective. +# +# In case a separate license agreement ends, and such agreement ends without being +# replaced by another separate license agreement, the license terms below apply +# from the time at which said agreement ends. +# +# LICENSE TERMS +# +# This program is free software: you can redistribute it and/or modify it under the +# terms of the GNU Affero General Public License, version 3, as published by the +# Free Software Foundation. This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See the GNU Affero General Public License Version 3 for more details. +# +# You should have received a copy of the GNU Affero General Public license along +# with this program. If not, see . +# +##################################################################################### + +if __name__ == '__main__': + from pkg_resources import load_entry_point + import sys + + sys.exit( + load_entry_point('crossbar', 'console_scripts', 'crossbar')() + ) From 0fe825a1f6cb2c7a8805ad4217906a1f9beea1cb Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Mon, 21 Nov 2016 12:46:45 +1100 Subject: [PATCH 44/48] update license --- crossbar/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crossbar/__main__.py b/crossbar/__main__.py index a253cc467..3244d3382 100644 --- a/crossbar/__main__.py +++ b/crossbar/__main__.py @@ -1,9 +1,9 @@ ##################################################################################### # -# Copyright (C) Tavendo GmbH +# Copyright (c) Crossbar.io Technologies GmbH # -# Unless a separate license agreement exists between you and Tavendo GmbH (e.g. you -# have purchased a commercial license), the license terms below apply. +# Unless a separate license agreement exists between you and Crossbar.io GmbH (e.g. +# you have purchased a commercial license), the license terms below apply. # # Should you enter into a separate license agreement after having received a copy of # this software, then the terms of such license agreement replace the terms below at From 6d2833e1aacc9b742eda50433f82ebd623225258 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Tue, 29 Nov 2016 21:34:40 +1100 Subject: [PATCH 45/48] bump dependencies --- requirements.txt | 153 ++++++++++++++++++++++++----------------------- 1 file changed, 77 insertions(+), 76 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1d16bb548..44c3d85b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ argh==0.26.2 \ --hash=sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3 \ --hash=sha256:e9535b8c84dc9571a48999094fda7f33e63c3f1b74f3e5f3ac0105a58405bb65 -attrs==16.2.0 \ - --hash=sha256:ce9d6cac4705e5aeaca02d3ff72f0006bf9b0a2f29635ae8dab8262e296f6442 \ - --hash=sha256:136f2ec0f94ec77ff2990830feee965d608cab1e8922370e3abdded383d52001 +attrs==16.3.0 \ + --hash=sha256:c59426b15b45e39a7bc408eb6ba7e7188d9532764f873cc691199ddd975c97ef \ + --hash=sha256:80203177723e36f3bbe15aa8553da6e80d47bfe53647220ccaa9ad7a5e473ccc autobahn==0.16.1 \ --hash=sha256:f6b38538aa2ff911e1cc687be6ba183465974c5bdbfdd00437bd8300dcb838d5 \ --hash=sha256:d4139862620bab15f1a9711e83430d0b20484d713bc0ced51185a921b9990375 @@ -11,71 +11,72 @@ bitstring==3.1.5 \ --hash=sha256:c163a86fcef377c314690051885d86b47419e3e1770990c212e16723c1c08faa cbor==1.0.0 \ --hash=sha256:13225a262ddf5615cbd9fd55a76a0d53069d18b07d2e9f19c39e6acb8609bbb6 -cffi==1.8.3 \ - --hash=sha256:d301161bbabdc17d61886e529fb9ea8305cbf2f3f03a86831bd77144c5e9a781 \ - --hash=sha256:1090590fbeb8f369251c822c3e6516b891b6fdd577625d71bab31bacbd304c77 \ - --hash=sha256:24e47ebf130ed26b970bcd2bc2436cdebffb24a1cf4a9cd95c5306b67439f367 \ - --hash=sha256:7cbba4967c04b0940de743c4e3ecb82274ba5568fc100360aeae78d83921b34e \ - --hash=sha256:4ff2b8650bc0fa499d24dbbe85b41faffe2c4c1a4fc241edf075443a5c519591 \ - --hash=sha256:c46eab25dcceb4ef98094a1bc5bd3d6a3fd4429b27e0a53dc9e86b9e7375a2d7 \ - --hash=sha256:2e882a295f17cb13f992bde07647e05af7fe91fa5854d3f10cad8c43401e0522 \ - --hash=sha256:74866d355ea3afe8b33f735ac4d14d485815e6d189a679f4fb2348c0113063fb \ - --hash=sha256:1ee7295bf61ed60eb544970484eeb47a17be64e1596654ab4614a038688e5430 \ - --hash=sha256:0ee962d901fdd5afe53e6c609edca76427cd5b9de6945fd66c5fc0fc23c19017 \ - --hash=sha256:d64b7084a0e97903c779f19605236eddcb4ec27fbe25c16bad3b96c275384325 \ - --hash=sha256:e68f4f39c079ad3819b653fcc2dfa44f234a037473f19d6e7d1685d8de7d24f6 \ - --hash=sha256:7fe6eb09ec7db1c269040cb92e5167fafaa5da747a6400d170f9051c52f08730 \ - --hash=sha256:2d92b3d661211ceac94aca2a4efda4db709174e04c9a30ed12fb8ab78e78672f \ - --hash=sha256:b56289f06c124ed191081df55daeb37b571ade9375d1e18988d0c9819e4a675f \ - --hash=sha256:a7f917d784d51210516ed71b28d4994887be8ed2d99754c1bb49b12aa98bd5fa \ - --hash=sha256:47ea859cd153300778f33220c69cad97cc9c4d3e312a50b01d1521b94789ebfc \ - --hash=sha256:91757b9cfc1b47ffd563ce46090b2494eaec7a1ffd7f41a6f1771dc199f21115 \ - --hash=sha256:d0140dbca2134825b3f38024d4e140a2ad9baeb6ee8f87fd48a442062f417ad0 \ - --hash=sha256:ba1faa81f4e6daa8025e52ed249dcb36ae27ef00047499e1741896688433c146 \ - --hash=sha256:d9c345d966685cd558b7f2d1ec6520cca1823e36c0bad89692808f13bdb5e12f \ - --hash=sha256:9dcc07d7d5b786329088461498523514cd51adab02a3e8ca7f764cad8b9aa50e \ - --hash=sha256:9e466d8dd18ab0ee8def4e2c5df2bf0892957c9276b6eb406d8efcc632f3ded7 \ - --hash=sha256:45445bcd19a01fcd33041cbf1aa997216296d44a32d892b0ae780dc2644258b5 \ - --hash=sha256:6f2802177893c97beeddbacb96bded0db411ccd6b05b020eebcd42d20e2b96d8 \ - --hash=sha256:712b4cc349a56a7d5895791cda291ffba8da407375098f283a6496d2b088f6e7 \ - --hash=sha256:8dafe0d5a19e039a4ee8bc6070d2136e9373ea85fbdd3e2b2cab21f7d60c77e7 \ - --hash=sha256:91517da73b86cbfe1911bc5f1e0c66aa270d0d73b6543be9aa0befa12d9239d3 \ - --hash=sha256:828cdf093ad07b41d8005c8bafd1cd197a9b0aae33fc2f5b52ddbdb4bec765b1 \ - --hash=sha256:6ff29804567502cdde5b6f2efac1caaaae62a0232d21f4d39a2ca744c6a3df7c \ - --hash=sha256:b7a146d7229e0c3e9818c4240eade4430d0af88495037d27085374abef23ff83 \ - --hash=sha256:47e0337a814afd05257ce5bfede318b1dcabcc90e3c50906643e0aa17dab738c \ - --hash=sha256:75348f45112cee84c233a010f404b9369da438e578104dd3b01f1054f9f7519a \ - --hash=sha256:f76b68ba5f90a60a6cc319efd45c46a578ecf15d6d10a1f8055f59da6af77d38 \ - --hash=sha256:f48ec170fa1d6e388a3162af000b76e84455f4c29cd62b08d64f49bf2a98581c \ - --hash=sha256:c321bd46faa7847261b89c0469569530cad5a41976bb6dba8202c0159f476568 +cffi==1.9.1 \ + --hash=sha256:6120b62a642a40e47eb6c9ff00c02be69158fc7f7c5ff78e42a2c739d1c57cd6 \ + --hash=sha256:bcaf3d86385daaab0ae51c9c53ebe70a6c1c5dfcb9e311b13517e04773ddf6b6 \ + --hash=sha256:6fbf8db55710959344502b58ab937424173ad8b5eb514610bcf56b119caa350a \ + --hash=sha256:f8ba54848dfe280b1be0d6e699544cee4ba10d566f92464538063d9e645aed3e \ + --hash=sha256:f8264463cc08cd696ad17e4bf3c80f3344628c04c11ffdc545ddf0798bc17316 \ + --hash=sha256:04b133ef629ae2bc05f83d0b079a964494a9cd17914943e690c57209b44aae20 \ + --hash=sha256:fde17c52d7ce7d55a9fb263b57ccb5da6439915b5c7105617eb21f636bb1bd9c \ + --hash=sha256:3f1908d0bcd654f8b7b73204f24336af9f020b707fb8af937e3e2279817cbcd6 \ + --hash=sha256:d3e3063af1fa6b59e255da9a812891cdaf24b90fbaf653c02797871069b7c4c9 \ + --hash=sha256:f1366150acf611d09d37ffefb3559ed3ffeb1713643d3cd10716d6c5da3f83fb \ + --hash=sha256:d9cfe26ecea2fec320cd0cac400c9c2435328994d23596ee6df63945fe7292b0 \ + --hash=sha256:b0bc2d83cc0ba0e8f0d9eca2ffe07f72f33bec7d84547071e7e875d4cca8272d \ + --hash=sha256:cfa15570ecec1ea6bee089e86fd4deae6208c96a811344ce246de5e5c9ac824a \ + --hash=sha256:fce6b0cb9ade1546178c031393633b09c4793834176496c99a94de0bfa471b27 \ + --hash=sha256:36d06de7b09b1eba54b1f5f76e2221afef7489cc61294508c5a7308a925a50c6 \ + --hash=sha256:20af85d8e154b50f540bc8d517a0dbf6b1c20b5d06e572afda919d5dafd1d06b \ + --hash=sha256:9163f7743cf9991edaddf9cf886708e288fab38e1b9fec9c41c15c85c8f7f147 \ + --hash=sha256:ada8a42c493e4934a1a8875c2bc9efcb1b88c09883f70375bfa053ab32d6a118 \ + --hash=sha256:f93d1edcaea7b6a7a8fbf936f4492a9a0ee0b4cb281efebd5e1dd73e5e432c71 \ + --hash=sha256:5e1368d13f1774852f9e435260be19ad726bbfb501b80472f61c2dc768a0692a \ + --hash=sha256:74aadea668c94eef4ceb09be3d0eae6619e28b4f1ced4e29cd43a05bb2cfd7a4 \ + --hash=sha256:31776a37a67424e7821324b9e03a05aa6378bbc2bccc58fa56402547f82803c6 \ + --hash=sha256:2570f93b42c61013ab4b26e23aa25b640faf5b093ad7dd3504c3a8eadd69bc24 \ + --hash=sha256:fc8865c7e0ac25ddd71036c2b9a799418b32d9acb40400d345b8791b6e1058cb \ + --hash=sha256:97d9f338f91b7927893ea6500b953e4b4b7e47c6272222992bb76221e17056ff \ + --hash=sha256:353421c76545f1d440cacc137abc865f07eab9df0dd3510c0851a2ca04199e90 \ + --hash=sha256:83266cdede210393889471b0c2631e78da9d4692fcca875af7e958ad39b897ee \ + --hash=sha256:7be1efa623e1ed91b15b1e62e04c536def1d75785eb930a0b8179ca6b65ed16d \ + --hash=sha256:65c223e77f87cb463191ace3398e0a6d84ce4ac575d42eb412a220b099f593d6 \ + --hash=sha256:2f4e2872833ee3764dfc168dea566b7dd83b01ac61b377490beba53b5ece57f7 \ + --hash=sha256:f4eb9747a37120b35f59c8e96265e87b0c432ff010d32fc0772992aa14659502 \ + --hash=sha256:5268de3a18f031e9787c919c1b9137ff681ea696e76740b1c6c336a26baaa58a \ + --hash=sha256:a7930e73a4359b52323d09de6d6860840314aa09346cbcf4def8875e1b07ebc7 \ + --hash=sha256:ba6b5205fced1625b6d9d55f9ef422f9667c5d95f18f07c0611eb964a3355331 \ + --hash=sha256:1fb1cf40c315656f98f4d3acfb1bd031a14a9a69d155e9a180d5f9b52eaf745a \ + --hash=sha256:563e0bd53fda03c151573217b3a49b3abad8813de9dd0632e10090f6190fdaf8 click==6.6 \ + --hash=sha256:fcf697e1fd4b567d817c69dab10a4035937fe6af175c05fd6806b69f74cbc6c4 \ --hash=sha256:cc6a19da8ebff6e7074f731447ef7e112bd23adf3de5c597cf9989f2fd8defe9 constantly==15.1.0 \ --hash=sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d \ --hash=sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35 -cryptography==1.5.3 \ - --hash=sha256:e514d92086246b53ae9b048df652cf3036b462e50a6ce9fac6b6253502679991 \ - --hash=sha256:10ee414f4b5af403a0d8f20dfa80f7dad1fc7ae5452ec5af03712d5b6e78c664 \ - --hash=sha256:7234456d1f4345a144ed07af2416c7c0659d4bb599dd1a963103dc8c183b370e \ - --hash=sha256:d3b9587406f94642bd70b3d666b813f446e95f84220c9e416ad94cbfb6be2eaa \ - --hash=sha256:b15fc6b59f1474eef62207c85888afada8acc47fae8198ba2b0197d54538961a \ - --hash=sha256:3b62d65d342704fc07ed171598db2a2775bdf587b1b6abd2cba2261bfe3ccde3 \ - --hash=sha256:059343022ec904c867a13bc55d2573e36c8cfb2c250e30d8a2e9825f253b07ba \ - --hash=sha256:c7897cf13bc8b4ee0215d83cbd51766d87c06b277fcca1f9108595508e5bcfb4 \ - --hash=sha256:9b69e983e5bf83039ddd52e52a28c7faedb2b22bdfb5876377b95aac7d3be63e \ - --hash=sha256:61e40905c426d02b3fae38088dc66ce4ef84830f7eb223dec6b3ac3ccdc676fb \ - --hash=sha256:00783a32bcd91a12177230d35bfcf70a2333ade4a6b607fac94a633a7971c671 \ - --hash=sha256:d11973f49b648cde1ea1a30e496d7557dbfeccd08b3cd9ba58d286a9c274ff8e \ - --hash=sha256:f24bedf28b81932ba6063aec9a826669f5237ea3b755efe04d98b072faa053a5 \ - --hash=sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ - --hash=sha256:8a53209de822e22b5f73bf4b99e68ac4ccc91051fd6751c8252982983e86a77d \ - --hash=sha256:5a07439d4b1e4197ac202b7eea45e26a6fd65757652dc50f1a63367f711df933 \ - --hash=sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ - --hash=sha256:eaa4a7b5a6682adcf8d6ebb2a08a008802657643655bb527c95c8a3860253d8e \ - --hash=sha256:8156927dcf8da274ff205ad0612f75c380df45385bacf98531a5b3348c88d135 \ - --hash=sha256:61ec0d792749d0e91e84b1d58b6dfd204806b10b5811f846c2ceca0de028c53a \ - --hash=sha256:26330c88041569ca621cc42274d0ea2667a48b6deab41467272c3aba0b6e8f07 \ - --hash=sha256:cf82ddac919b587f5e44247579b433224cc2e03332d2ea4d89aa70d7e6b64ae5 +cryptography==1.6 \ + --hash=sha256:e831efaba4f48649c4f1fab21eab4b21fe350d42dfd4c824f186dd8f93501804 \ + --hash=sha256:f05445e5d4e0619f0652ff972fcb6aedc5600921e80435ec5d1b8df0d1b6e5ac \ + --hash=sha256:d0076e16ede4c60a16677b4130ceaaf147f9012640356656741e51f74229b15a \ + --hash=sha256:f2cd75336b52271a355631331cdd740d37dfdbe4ebf1c505acbbde5e861f7ac4 \ + --hash=sha256:be2f681b68f983278c07bf38134bc791f64603b0e1146351d110b3c420536943 \ + --hash=sha256:c56a10ff0c39350c98912cc0c266d65c07136e2aa285914f41af88d19fade0b2 \ + --hash=sha256:612b3d67628129d892100afde5a4a1d006416116159a2952f00e615e8203229e \ + --hash=sha256:e823f0443926446e134937dd82529a040c32778a0a6031f4dce5fb61df35dca0 \ + --hash=sha256:ea2d903e7c36bef0754640e8aef0cf6e028669b11c1524c5de13cc9ac23c5be6 \ + --hash=sha256:8fe0f14a51fc35e59df6db75a9b2157a3ea3316afeb699bb0fe341f2b326efd4 \ + --hash=sha256:142f066d50ca360771b71ae59a0650ca079dc477d51f40de10105696af3127d7 \ + --hash=sha256:f0609f3ded8095c763ac052a6f7ef44e8fc0135669d30d9f7d37ba4cca6d06e5 \ + --hash=sha256:ab706bb1c3ed4065b6139ed58df1988503a067a37b377f0594e49cdbc4fe349d \ + --hash=sha256:36e9359a9378f3bba3be7a798a1e9aeea752aaf9c1fb7ed6901ad22c7d0f4dee \ + --hash=sha256:495ccb3af051f487f8f27812b10b227e2b51ea510ba41e14532e16edb9aefdb5 \ + --hash=sha256:fc010b3a8191e07cf5a4e1bcb405d85f9a18a1a5199c21271eaf448333d8c160 \ + --hash=sha256:aedcf324cf9e2b43c2061a3eb0f29057399607794239f23df120819129705a0b \ + --hash=sha256:f9894612dbc0fb7cb6476a97a335fd1621f10d426b9d26203e8e58c1ac2dc02a \ + --hash=sha256:14de89e43d0d68fe5b6a6224dbcd956913a7a4a6f5c34a8dc90981b1da887485 \ + --hash=sha256:fa0acaa2420ba3db05df6e98256c01475f5183f3e9103a5fde08ca29ec7b9c76 \ + --hash=sha256:96bf30786292843c516ea7074dba33cf4ed2540c68da3e49ec24517de00e37b9 \ + --hash=sha256:4d0d86d2c8d3fc89133c3fa0d164a688a458b6663ab6fa965c80d6c2cdaf9b3f enum34==1.1.6 \ --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \ --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \ @@ -112,7 +113,7 @@ lmdb==0.92 \ --hash=sha256:d6caa0f578c8e301e1d761bbb8c3f5532bace43cbc80dfa141ff8913a353baad \ --hash=sha256:819bf3f9d403301c8c988eb572c2a07afdc96291b3fafd46b938975188a37088 \ --hash=sha256:d08e4aefa3f60c0c48e3df6ca92622fe1bb265b9341d25465b0ee2882d4024e3 \ - --hash=sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ + --hash=sha256:2b8bd1ff5536426417512fd493719fa0be3319875c37c940ad243835165c73dc \ --hash=sha256:00fd120160388d22a2dea8750beb8be846dfb439c76a095753d4caa96fa64a70 \ --hash=sha256:6c557a927763115512bd95d8527ab45095a3d9c6392bd0319d3265e5014102a1 \ --hash=sha256:596ab7102245831f1398eaa12beb4ca037187121821243794d08b6d6936b0a84 \ @@ -125,8 +126,8 @@ mistune==0.7.3 \ netaddr==0.7.18 \ --hash=sha256:cb305179658334eb035860e515f054504e232b832abb4efc51c04bf8a72d3574 \ --hash=sha256:98c3d6fe831d119785c37af25f032169dd653a0dad86e54b387f4a5b83da4383 \ - --hash=sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 \ - --hash=sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + --hash=sha256:a1f5c9fcf75ac2579b9995c843dade33009543c04f218ff7c007b3c81695bd19 \ + --hash=sha256:c64c570ac612e20e8b8a6eee72034c924fff9d76c7a46f50a9f919085f1bfbed pathtools==0.1.2 \ --hash=sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0 psutil==5.0.0 \ @@ -215,9 +216,9 @@ PyYAML==3.12 \ --hash=sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8 \ --hash=sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6 \ --hash=sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7 -requests==2.11.1 \ - --hash=sha256:545c4855cd9d7c12671444326337013766f4eea6068c3f0307fb2dc2696d580e \ - --hash=sha256:5acf980358283faba0b897c73959cecf8b841205bb4b2ad3ef545f46eae1a133 +requests==2.12.1 \ + --hash=sha256:3f3f27a9d0f9092935efc78054ef324eb9f8166718270aefe036dfa1e4f68e1e \ + --hash=sha256:2109ecea94df90980be040490ff1d879971b024861539abb00054062388b612e sdnotify==0.3.1 \ --hash=sha256:e69220d4f6cbb02130f43f929350a80cf51033fde47dcb056fbda71e2dff2d5a service_identity==16.0.0 \ @@ -226,10 +227,10 @@ service_identity==16.0.0 \ setproctitle==1.1.10 \ --hash=sha256:6283b7a58477dd8478fbb9e76defb37968ee4ba47b05ec1c053cb39638bd7398 \ --hash=sha256:6a035eddac62898786aed2c2eee7334c28cfc8106e8eb29fdd117cac56c6cdf0 -setuptools==28.8.0 \ - --hash=sha256:2e59d06ac798efce29c567ee0e0687514efca529a665b8f364cf497cd10d21b2 \ - --hash=sha256:432a1ad4044338c34c2d09b0ff75d509b9849df8cf329f4c1c7706d9c2ba3c61 \ - --hash=sha256:d41de15d58dc01d022f0aff6b514c20709d6f30948d9bfd8581960f3d5e7b997 +setuptools==29.0.1 \ + --hash=sha256:2b55dc2fe3cc72b5d6769f6099af35b061d5a76b4a907cadf8aaae4b41176252 \ + --hash=sha256:b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0 \ + --hash=sha256:c9c24ebdfe7b83fe90c4fe503b4152d721ef17e13714200c8e9ddfeed27fefd8 shutilwhich==1.1.0 \ --hash=sha256:db1f39c6461e42f630fa617bb8c79090f7711c9ca493e615e43d0610ecb64dc6 six==1.10.0 \ @@ -238,9 +239,9 @@ six==1.10.0 \ treq==15.1.0 \ --hash=sha256:1ad1ba89ddc62ae877084b290bd327755b13f6e7bc7076dc4d8e2efb701bfd63 \ --hash=sha256:425a47d5d52a993d51211028fb6ade252e5fbea094e878bb4b644096a7322de8 -Twisted==16.5.0 \ - --hash=sha256:b3594162580b8706073490b44311cfdbb78dca2ddfa3ef791269d924b01db6f0 \ - --hash=sha256:0207d88807482fa670a84926590e163a2a081a29745de34c5a6dc21066abae73 +Twisted==16.6.0 \ + --hash=sha256:ed19c3cd92e94798ea79f06f939159f8858a3a4b65a26c40b275369d8cf5ac3e \ + --hash=sha256:d0fe115ea7ef8cf632d05103de60356c6e992b2153d6830bdc4476f8accb1fca txaio==2.5.2 \ --hash=sha256:161494c82cf031afb2bc32277fd70be664029e785d8d12ac8d579d257abbfd6d \ --hash=sha256:321d441b336447b72dbe81a4d73470414454baf0543ec701fcfecbf4dcbda0fe From b1593dd1b6a4518a64b0332c2d15b944106c9325 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Wed, 30 Nov 2016 12:12:05 +1100 Subject: [PATCH 46/48] use a non-broken flake8, and actually check the dir we should --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index fceacf737..4dfbe738f 100644 --- a/tox.ini +++ b/tox.ini @@ -6,8 +6,9 @@ exclude = crossbar/worker/test/examples/syntaxerror.py [testenv:flake8] skip_install = True +changedir={toxinidir} deps = - flake8 + flake8==3.2.0,<3.2.1 commands = flake8 --ignore=E402,F405,E501,E731,N801,N802,N803,N805,N806 crossbar basepython = python2.7 From 1a41f863e8bba68415f256b7a6e11ac5fc2b8156 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Tue, 29 Nov 2016 21:18:11 +1100 Subject: [PATCH 47/48] unrelated pyflakes fixes --- crossbar/common/process.py | 8 ++++---- crossbar/router/protocol.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crossbar/common/process.py b/crossbar/common/process.py index 13fd4d41f..c57f80814 100644 --- a/crossbar/common/process.py +++ b/crossbar/common/process.py @@ -567,6 +567,10 @@ def start_manhole(self, config, details=None): # namespace = {'session': self} + from twisted.conch.manhole_ssh import ( + ConchFactory, TerminalRealm, TerminalSession) + from twisted.conch.manhole import ColoredManhole + class PatchedTerminalSession(TerminalSession): # get rid of # exceptions.AttributeError: TerminalSession instance has no attribute 'windowChanged' @@ -574,10 +578,6 @@ class PatchedTerminalSession(TerminalSession): def windowChanged(self, winSize): pass - from twisted.conch.manhole_ssh import ( - ConchFactory, TerminalRealm, TerminalSession) - from twisted.conch.manhole import ColoredManhole - rlm = TerminalRealm() rlm.sessionFactory = PatchedTerminalSession # monkey patch rlm.chainedProtocolFactory.protocolFactory = lambda _: ColoredManhole(namespace) diff --git a/crossbar/router/protocol.py b/crossbar/router/protocol.py index ded41e45d..a9d54279d 100644 --- a/crossbar/router/protocol.py +++ b/crossbar/router/protocol.py @@ -244,7 +244,7 @@ def print_traffic(): # so immediately authenticate the client using that information self._authprovider = u'cookie' self.log.debug("Authenticated client via cookie {cookiename}={cbtid} as authid={authid}, authrole={authrole}, authmethod={authmethod}, authrealm={authrealm}", - cookiename=self.factory._cookiestore._cookie_id_field, cbtid=self._cbtid, authid=self._authid, authrole=self._authrole, authmethod=self._authmethod, authrealm=self._authrealm) + cookiename=self.factory._cookiestore._cookie_id_field, cbtid=self._cbtid, authid=self._authid, authrole=self._authrole, authmethod=self._authmethod, authrealm=self._authrealm) else: # there is a cookie set, but the cookie wasn't authenticated yet using a different auth method self.log.debug("Cookie-based authentication enabled, but cookie isn't authenticated yet") From 8da1e901acd2483fde03e3f58492ea71d2993f37 Mon Sep 17 00:00:00 2001 From: "Amber Brown (HawkOwl)" Date: Wed, 30 Nov 2016 12:21:28 +1100 Subject: [PATCH 48/48] flake8 fixes --- crossbar/_log_categories.py | 9 --------- crossbar/adapter/mqtt/_events.py | 1 + crossbar/adapter/mqtt/protocol.py | 1 + crossbar/adapter/mqtt/test/interop.py | 8 +------- crossbar/adapter/mqtt/test/interop_tests.py | 1 - crossbar/adapter/mqtt/test/test_tx.py | 13 +------------ crossbar/adapter/mqtt/tx.py | 2 +- crossbar/common/checkconfig.py | 5 +++++ 8 files changed, 10 insertions(+), 30 deletions(-) diff --git a/crossbar/_log_categories.py b/crossbar/_log_categories.py index 6c4ab0576..e3df23792 100644 --- a/crossbar/_log_categories.py +++ b/crossbar/_log_categories.py @@ -74,15 +74,6 @@ "MQ202": "Received a QoS 1 Publish from '{client_id}'", "MQ203": "Received a QoS 2 Publish from '{client_id}'", "MQ204": "Received a Disconnect from '{client_id}', closing connection", - "MQ300": "Got a PubACK for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", - "MQ301": "Got a PubREC for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", - "MQ302": "Got a PubREC for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", - "MQ302": "Got a PubCOMP for publish ID {pub_id} from '{client_id}' that doesn't correspond to any Publish we sent", - "MQ303": "Client '{client_id}' sent a PubACK for a non-QoS 1 Publish", - "MQ304": "Client '{client_id}' sent a PubREC for a non-QoS 2 Publish", - "MQ305": "Client '{client_id}' sent a duplicate PubREC", - "MQ306": "Client '{client_id}' sent a PubCOMP for a non-QoS 2 Publish", - "MQ307": "Client '{client_id}' sent a PubCOMP before a PubREC", "MQ303": "Got a non-allowed QoS value in the publish queue, dropping it.", "MQ400": "MQTT client '{client_id}' timed out after recieving no full packets for {seconds}", "MQ401": "Protocol violation from '{client_id}', terminating connection: {error}", diff --git a/crossbar/adapter/mqtt/_events.py b/crossbar/adapter/mqtt/_events.py index 118494732..b07031031 100644 --- a/crossbar/adapter/mqtt/_events.py +++ b/crossbar/adapter/mqtt/_events.py @@ -241,6 +241,7 @@ def deserialise(cls, flags, data): return cls(packet_identifier) + @attr.s class PubREC(object): packet_identifier = attr.ib(validator=instance_of(int)) diff --git a/crossbar/adapter/mqtt/protocol.py b/crossbar/adapter/mqtt/protocol.py index 0d1a49bc7..bebb3592f 100644 --- a/crossbar/adapter/mqtt/protocol.py +++ b/crossbar/adapter/mqtt/protocol.py @@ -52,6 +52,7 @@ class _NeedMoreData(Exception): We need more data before we can get the bytes length. """ + # State machine events WAITING_FOR_NEW_PACKET = 0 COLLECTING_REST_OF_PACKET = 1 diff --git a/crossbar/adapter/mqtt/test/interop.py b/crossbar/adapter/mqtt/test/interop.py index 1c811f293..af2f6cbad 100644 --- a/crossbar/adapter/mqtt/test/interop.py +++ b/crossbar/adapter/mqtt/test/interop.py @@ -43,7 +43,6 @@ from texttable import Texttable from twisted.internet.protocol import Protocol, ClientFactory -from twisted.logger import globalLogBeginner, textFileLogObserver from crossbar.adapter.mqtt.protocol import MQTTClientParser @@ -52,6 +51,7 @@ class Frame(object): send = attr.ib() data = attr.ib() + class ConnectionLoss(object): send = False data = b"" @@ -70,8 +70,6 @@ class Result(object): @click.option("--port") def run(host, port): - #globalLogBeginner.beginLoggingTo([textFileLogObserver(sys.stdout)]) - port = int(port) from . import interop_tests @@ -106,8 +104,6 @@ def run(host, port): sys.exit(0) - - class ReplayProtocol(Protocol): def __init__(self, factory): @@ -176,7 +172,6 @@ def wait(): self._waiting_for_nothing = self.factory.reactor.callLater(2, wait) return - def connectionLost(self, reason): if self.factory.reactor.running: if self._record and isinstance(self._record[0], ConnectionLoss): @@ -196,7 +191,6 @@ class ReplayClientFactory(ClientFactory): protocol = ReplayProtocol noisy = False - def buildProtocol(self, addr): self.client_transcript = [] diff --git a/crossbar/adapter/mqtt/test/interop_tests.py b/crossbar/adapter/mqtt/test/interop_tests.py index 321f35388..f4e75ce9d 100644 --- a/crossbar/adapter/mqtt/test/interop_tests.py +++ b/crossbar/adapter/mqtt/test/interop_tests.py @@ -286,7 +286,6 @@ def test_qos2_send_wrong_confirm(host, port): return Result("qos2_wrong_confirm", f.success, f.reason, f.client_transcript) - def test_qos1_send_wrong_confirm(host, port): record = [ Frame( diff --git a/crossbar/adapter/mqtt/test/test_tx.py b/crossbar/adapter/mqtt/test/test_tx.py index af198fd94..797e4f66e 100644 --- a/crossbar/adapter/mqtt/test/test_tx.py +++ b/crossbar/adapter/mqtt/test/test_tx.py @@ -36,8 +36,7 @@ from crossbar.adapter.mqtt.tx import ( MQTTServerTwistedProtocol, Session, Message) -from crossbar.adapter.mqtt.protocol import ( - MQTTParser, MQTTClientParser, client_packet_handlers, P_CONNACK) +from crossbar.adapter.mqtt.protocol import MQTTClientParser from crossbar.adapter.mqtt._events import ( Connect, ConnectFlags, ConnACK, SubACK, Subscribe, @@ -476,7 +475,6 @@ def test_unknown_connect_code_must_lose_connection(self): Compliance statements MQTT-3.2.2-4, MQTT-3.2.2-5 """ - d = Deferred() h = BasicHandler(6) sessions, r, t, p, cp = make_test_items(h) @@ -1168,8 +1166,6 @@ def test_qos_0_queues_message(self): The WAMP layer calling send_publish will queue a message up for sending, and send it next time it has a chance. """ - got_packets = [] - h = BasicHandler() sessions, r, t, p, cp = make_test_items(h) @@ -1214,8 +1210,6 @@ def test_qos_1_queues_message(self): The WAMP layer calling send_publish will queue a message up for sending, and send it next time it has a chance. """ - got_packets = [] - h = BasicHandler() sessions, r, t, p, cp = make_test_items(h) @@ -1621,8 +1615,6 @@ def test_non_allowed_qos_not_queued(self): """ A non-QoS 0, 1, or 2 message will be rejected by the publish layer. """ - got_packets = [] - h = BasicHandler() sessions, r, t, p, cp = make_test_items(h) @@ -1662,8 +1654,6 @@ def test_non_allowed_qos_in_queue_dropped(self): If a non-QoS 0, 1, or 2 message gets into the queue, it will be dropped. """ - got_packets = [] - h = BasicHandler() sessions, r, t, p, cp = make_test_items(h) @@ -1703,7 +1693,6 @@ def test_non_allowed_qos_in_queue_dropped(self): # Nothing sent self.assertEqual(t.value(), b'') - def test_does_not_schedule_if_disconnected(self): """ If a publish is sent whilst the client is disconnected, it won't be diff --git a/crossbar/adapter/mqtt/tx.py b/crossbar/adapter/mqtt/tx.py index 95f4de8f5..536c62b2c 100644 --- a/crossbar/adapter/mqtt/tx.py +++ b/crossbar/adapter/mqtt/tx.py @@ -135,7 +135,7 @@ def connectionLost(self, reason): def send_publish(self, topic, qos, body): - if not qos in [0, 1, 2]: + if qos not in [0, 1, 2]: raise ValueError("QoS must be [0, 1, 2]") self.session.queued_messages.append(Message(topic=topic, qos=qos, body=body)) diff --git a/crossbar/common/checkconfig.py b/crossbar/common/checkconfig.py index 641d3e20e..f90a39bce 100644 --- a/crossbar/common/checkconfig.py +++ b/crossbar/common/checkconfig.py @@ -163,6 +163,7 @@ def pprint_json(obj, log_to=None): def construct_yaml_str(self, node): return self.construct_scalar(node) + for Klass in [Loader, SafeLoader]: Klass.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) @@ -184,6 +185,7 @@ def construct_ordered_mapping(self, node, deep=False): mapping[key] = value return mapping + yaml.constructor.BaseConstructor.construct_mapping = construct_ordered_mapping @@ -193,6 +195,7 @@ def construct_yaml_map_with_ordered_dict(self, node): value = self.construct_mapping(node) data.update(value) + for Klass in [Loader, SafeLoader]: Klass.add_constructor('tag:yaml.org,2002:map', construct_yaml_map_with_ordered_dict) @@ -226,6 +229,7 @@ def represent_ordered_dict(dump, tag, mapping, flow_style=None): node.flow_style = best_style return node + for Klass in [Dumper, SafeDumper]: Klass.add_representer(OrderedDict, lambda dumper, value: represent_ordered_dict(dumper, u'tag:yaml.org,2002:map', value)) @@ -312,6 +316,7 @@ def get_config_value(config, item, default=None): else: return default + _CONFIG_ITEM_ID_PAT_STR = "^[a-z][a-z0-9_]{2,11}$" _CONFIG_ITEM_ID_PAT = re.compile(_CONFIG_ITEM_ID_PAT_STR)