From 568a2d777cdacfb08c1ff7cb58f78c8525832ec1 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Wed, 19 Apr 2023 07:59:04 -0500 Subject: [PATCH 01/11] feat(rtsp_client): add python tests * Added simple RTSP client using native opencv videocapture to open the stream * Added RtspClient class which handles the RTSP / RTP / RTCP packets and properly handles the incoming RTP MJPEG stream (repackaging the split packets and adding the jpeg headers) to show each frame via opencv. --- .../rtsp_client/python/opencv_rtsp_client.py | 18 + components/rtsp_client/python/rtsp_client.py | 321 ++++++++++++++++++ 2 files changed, 339 insertions(+) create mode 100644 components/rtsp_client/python/opencv_rtsp_client.py create mode 100644 components/rtsp_client/python/rtsp_client.py diff --git a/components/rtsp_client/python/opencv_rtsp_client.py b/components/rtsp_client/python/opencv_rtsp_client.py new file mode 100644 index 0000000..8fc2548 --- /dev/null +++ b/components/rtsp_client/python/opencv_rtsp_client.py @@ -0,0 +1,18 @@ +import sys +import cv2 + +def stream(addr, port): + uri = f"rtsp://{addr}:{port}/mjpeg/1" + print(f"Opening URI: {uri}, press 'q' to quit") + vcap = cv2.VideoCapture(uri) + while(1): + ret, frame = vcap.read() + cv2.imshow('VIDEO', frame) + if cv2.waitKey(1) & 0xFF == ord('q'): + break + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: python ./opencv_rtsp_client
") + sys.exit(1) + stream(sys.argv[1], sys.argv[2]) diff --git a/components/rtsp_client/python/rtsp_client.py b/components/rtsp_client/python/rtsp_client.py new file mode 100644 index 0000000..ddf5b9a --- /dev/null +++ b/components/rtsp_client/python/rtsp_client.py @@ -0,0 +1,321 @@ +import socket +import sys +import threading + +import cv2 +import io +import struct +import numpy as np + +import io + +''' + +NOTE: This code is designed to handle MJPEG video streams over RTP/RTCP. + +Some useful references: +* https://www.rfc-editor.org/rfc/rfc2435 + RTP Payload Format for JPEG-compressed Video +* https://en.wikipedia.org/wiki/JPEG_File_Interchange_Format + JFIF - the JPEG File Interchange Format +* https://en.wikipedia.org/wiki/JPEG + Wikipedia page for JPEG + +The huffman tables are not transferred with the images, but the quantization +tables are. The rest of the JPEG header/data is stripped from the stream, such +that to properly display it, you need to rebuild the jpeg header data based on +the simplified RTP & JFIF header data. Once you've done that, the jpeg frames +can be decoded properly. + +Somewhat unrelated, you can convert mp4 to mjpeg and mp3: + +```bash +ffmpeg -i input.mp4 -vf "fps=30,scale=-1:176:flags=lanczos,crop=220:in_h:(in_w-220)/2:0" -q:v 9 220_30fps.mjpeg +# for MP3 +ffmpeg -i input.mp4 -ar 44100 -ac 1 -q:a 9 44100.mp3 +# for PCM +ffmpeg -i input.mp4 -f u16be -acodec pcm_u16le -ar 44100 -ac 1 44100_u16le.pcm +``` + +''' + +huffman_table = [ + # 1st table + # Default luminance DC Huffman table + 0xff, 0xc4, 0x00, 0x1f, 0x00, # header + 0x00, 0x01, 0x05, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x02, 0x03, + 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, + # 2nd table + # Default luminance AC Huffman table + 0xff, 0xc4, 0x00, 0xb5, 0x10, # header + 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01, 0x7d, 0x01, 0x02, + 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, + 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, + 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, + 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, + 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, + 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, + 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2, 0xe3, 0xe4, + 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, + +] + +class RtspClient: + def __init__(self, server, port): + self.server = server + self.port = port + self.cseq = 0 + self.session_id = "" + + def connect(self): + self.sock = socket.create_connection((self.server, self.port)) + self.send_request("OPTIONS", "*") + + def send_request(self, method, uri, headers=None): + if headers is None: + headers = {} + + request = f"{method} {uri} RTSP/1.0\r\n" + request += f"CSeq: {self.cseq}\r\n" + if self.session_id: + request += f"Session: {self.session_id}\r\n" + + for key, value in headers.items(): + request += f"{key}: {value}\r\n" + + request += "User-Agent: RtspClient\r\n" + request += "\r\n" + + self.sock.sendall(request.encode()) + response = self.sock.recv(4096) + print("Response:", response.decode()) + + self.cseq += 1 + + def describe(self, uri): + self.send_request("DESCRIBE", uri, {"Accept": "application/sdp"}) + + def setup(self, uri, transport): + self.send_request("SETUP", uri, {"Transport": transport}) + + def play(self, uri): + self.send_request("PLAY", uri) + + def pause(self, uri): + self.send_request("PAUSE", uri) + + def teardown(self, uri): + self.send_request("TEARDOWN", uri) + + def start_receiving_video_stream(self, rtp_port, rtcp_port): + self.rtp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.rtp_socket.bind(("0.0.0.0", rtp_port)) + + self.rtcp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.rtcp_socket.bind(("0.0.0.0", rtcp_port)) + + print(f"Listening for RTP packets on port {rtp_port}") + print(f"Listening for RTCP packets on port {rtcp_port}") + + # NOTE: right now the rtp_thread must be run in the main thread context + # (mac os cannot run cv2.imshow in another thread), and we are not + # receiving any RTCP packets, so we've simply stopped spawning + # those threads and are instead direclty running the rtp_thread's + # function + + # TODO: get threaded cv2.imshow working (or have it simply update the + # frame and have opencv show happen in main thread context) + + # rtp_thread = threading.Thread(target=self.handle_rtp_packet) + # rtcp_thread = threading.Thread(target=self.handle_rtcp_packet) + + # rtp_thread.start() + # rtcp_thread.start() + + # rtp_thread.join() + # rtcp_thread.join() + + self.handle_rtp_packet() + + def handle_rtp_packet(self): + # for this example we'll show the received video stream in an opencv + # window + buf = bytearray() + cv2.namedWindow('MJPEG Stream', cv2.WINDOW_NORMAL) + + while True: + # Process RTP packet in rtp_data + rtp_data, addr = self.rtp_socket.recvfrom(8192) + print(f"received rtp packet, len={len(rtp_data)}") + rtp_header = rtp_data[:12] + rtp_payload = rtp_data[12:] # NAL unit + + + version, payload_and_marker, seq, timestamp, ssrc = struct.unpack('>BBHII', rtp_header) + payload_type = (payload_and_marker & 0x7F) + marker_bit = (payload_and_marker & 0b10000000) >> 7 + + # print("\tRTP Header:", [hex(x) for x in list(rtp_header)]) + print(f"\tMarker: {marker_bit}, Payload Type: {payload_type}") + # print(f"Sequence number: {seq}, Timestamp: {timestamp}, SSRC: {ssrc}") + + jpeg_header = rtp_payload[:8] + type_specific, frag_offset, frag_type, q, width, height = struct.unpack('>B3s BBBB', rtp_payload[:8]) + frag_offset = int.from_bytes(frag_offset, byteorder='big') + q = int(q) + width = int(width) * 8 + height = int(height) * 8 + + # print("\tJPEG header:", [hex(x) for x in list(jpeg_header)]) + print(f"\tJpeg image: width={width}, height={height}, type_spec={type_specific}, q={q}, frag_type={frag_type}, frag_offset={frag_offset}") + + jpeg_data = rtp_payload[8:] + + if 64 <= frag_type <= 127: + # there must be a restart marker header + print("\tHave restart header") + restart_header = rtp_payload[8:12] + restart_interval = int((restart_header[0] << 8) | restart_header[1]) + f_bit = True if restart_header[2] & 0x80 else False + l_bit = True if restart_header[2] & 0x80 else False + restart_count = int(((restart_header[2] & 0x3F) << 8) | restart_header[3]) + print(f"\tRestart interval={restart_interval}, f={f_bit}, l={l_bit}, restart_count={restart_count}") + jpeg_data = rtp_payload[12:] + + if 128 <= q <= 255: + print("\tGetting quantization table header") + # bytes 8,9,10 are all 0 based on CStreamer.cpp lines 109-111 + num_quant_bytes = rtp_payload[11] + quant_size = 64 + expected_quant_bytes = 2 * quant_size + if num_quant_bytes != expected_quant_bytes: + print(f"Unexpected quant bytes: {num_quant_bytes}, expected {expected_quant_bytes}") + else: + q0_offset = 12 + q1_offset = q0_offset + quant_size + q1_end = q1_offset + quant_size + q0 = rtp_payload[q0_offset:q1_offset] + q1 = rtp_payload[q1_offset:q1_end] + jpeg_data = rtp_payload[q1_end:] + + if frag_offset == 0: + # Create a binary stream to construct the JPEG header + jpeg_header = io.BytesIO() + + # Start Of Image (SOI) marker + jpeg_header.write(b'\xFF\xD8') + + jfif_app0_marker = bytearray([ + 0xFF, 0xE0, # APP0 marker + 0x00, 0x10, # Length (16 bytes) + 0x4A, 0x46, 0x49, 0x46, 0x00, # JFIF identifier + 0x01, 0x02, # JFIF version 1.2 + 0x01, # Units: DPI + 0x00, 0x48, # Xdensity: 72 DPI + 0x00, 0x48, # Ydensity: 72 DPI + 0x00, 0x00 # No thumbnail (width 0, height 0) + ]) + jpeg_header.write(jfif_app0_marker) + + # Quantization table (DQT) marker for luminance + # marker(0xFFDB), size (0x0043 = 67), index (0x00) + jpeg_header.write(b'\xFF\xDB\x00\x43\x00') + jpeg_header.write(bytearray(q0)) + + # Quantization table (DQT) marker for chrominance + # marker(0xFFDB), size (0x0043 = 67), index (0x01) + jpeg_header.write(b'\xFF\xDB\x00\x43\x01') + jpeg_header.write(bytearray(q1)) + + # Frame header (SOF0) marker + sof0_marker = bytearray([ + 0xFF, 0xC0, # SOF0 marker + 0x00, 0x11, # Length (17 bytes) + 0x08, # Data precision: 8 bits + *height.to_bytes(2, 'big'), # 0x01, 0xE0, # Image height: 240 + *width.to_bytes(2, 'big'), # 0x01, 0xE0, # Image width: 240 + 0x03, # Number of components: 3 (YCbCr) + 0x01, 0x21, 0x00, # Component 1 (Y): horizontal sampling factor = 2, vertical sampling factor = 1, quantization table ID = 0 + 0x02, 0x11, 0x01, # Component 2 (Cb): horizontal sampling factor = 1, vertical sampling factor = 1, quantization table ID = 1 + 0x03, 0x11, 0x01 # Component 3 (Cr): horizontal sampling factor = 1, vertical sampling factor = 1, quantization table ID = 1 + ]) + jpeg_header.write(sof0_marker) + + jpeg_header.write(bytes(huffman_table)) + + # Scan header (SOS) marker + # marker(0xFFDA), size of SOS (0x000C), num components(0x03), + # component specification parameters, + # spectral selection (0x003F), + # successive appromiation parameters (0x00) + jpeg_header.write(b'\xFF\xDA\x00\x0C\x03\x01\x00\x02\x11\x03\x11\x00\x3F\x00') + + jpeg_header_bytes = bytearray(jpeg_header.getvalue()) + + print(f"\tAdded header of length {len(jpeg_header_bytes)}") + + buf = jpeg_header_bytes + + # make sure we add the actual jpeg segment data + buf.extend(jpeg_data) + + if marker_bit: + # Add the JPEG end marker (EOI) + buf.extend(b'\xFF\xD9') + print(f"Decoding image size={len(buf)}") + frame = cv2.imdecode(np.frombuffer(buf, dtype=np.uint8), cv2.IMREAD_COLOR) + if frame is not None: + print(f"Decoded frame: {frame.shape}\n\n") + # our images are flipped vertically, fix it :) + # 0 = vertical, 1 = horizontal, -1 = both vertical and horiztonal + frame = cv2.flip(frame, 0) + cv2.imshow('MJPEG Stream', frame) + if cv2.waitKey(1) & 0xFF == ord('q'): + break + + def handle_rtcp_packet(self): + while True: + rtcp_data, addr = self.rtcp_socket.recvfrom(8192) + print("Received rtcp packet:", rtcp_data) + # Process RTCP packet in rtcp_data + # ... + # The handle_rtcp_packet function currently does nothing, but you + # can implement it to process RTCP packets, such as sender reports + # or receiver reports, depending on your application requirements. + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: python rtsp_client.py ") + sys.exit(1) + + server, port = sys.argv[1], int(sys.argv[2]) + client = RtspClient(server, port) + client.connect() + + # Replace the following line with the actual RTSP URI you want to stream + rtsp_uri = f"rtsp://{server}:{port}/mjpeg/1" + + # Call DESCRIBE method to get SDP information + client.describe(rtsp_uri) + + # The following lines are placeholders for RTP and RTCP ports + # You should parse the RTSP SETUP response and set the RTP and RTCP ports accordingly + rtp_port = 5000 + rtcp_port = 5001 + + # Set up the transport header with the RTP and RTCP ports + transport_header = f"RTP/AVP;unicast;client_port={rtp_port}-{rtcp_port}" + client.setup(rtsp_uri, transport_header) + + # Start streaming + print("Streaming:", rtsp_uri) + client.play(rtsp_uri) + + # Start receiving video stream + client.start_receiving_video_stream(rtp_port, rtcp_port) From 7d1e23bb8dc8bc05b615921f6f4400b239f0e5fb Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 20 Apr 2023 13:20:21 -0500 Subject: [PATCH 02/11] refactored python client to have classes for encapsulating jpeg header, rtp packet, rtp jpeg packet, and jpeg frame. now it is much faster :) --- components/rtsp_client/python/rtsp_client.py | 307 ++++++++++++------- 1 file changed, 190 insertions(+), 117 deletions(-) diff --git a/components/rtsp_client/python/rtsp_client.py b/components/rtsp_client/python/rtsp_client.py index ddf5b9a..8b80569 100644 --- a/components/rtsp_client/python/rtsp_client.py +++ b/components/rtsp_client/python/rtsp_client.py @@ -66,6 +66,171 @@ ] +class RtpPacket: + def __init__(self, data): + self.data = data + rtp_info, payload_and_marker, self.sequence_number, self.timestamp, self.ssrc = struct.unpack('>BBHII', data[:12]) + self.version = (rtp_info & 0b11000000) >> 6 + self.padding = (rtp_info & 0b00100000) >> 5 + self.extension = (rtp_info & 0b00010000) >> 4 + self.csrc_count = rtp_info & 0b00001111 + self.payload_type = (payload_and_marker & 0x7F) + self.marker = (payload_and_marker & 0b10000000) >> 7 + + # self.version = (data[0] & 0b11000000) >> 6 + # self.padding = (data[0] & 0b00100000) >> 5 + # self.extension = (data[0] & 0b00010000) >> 4 + # self.csrc_count = data[0] & 0b00001111 + # self.marker = (data[1] & 0b10000000) >> 7 + # self.payload_type = data[1] & 0b01111111 + # self.sequence_number = data[2] * 256 + data[3] + # self.timestamp = data[4] * 256 * 256 * 256 + data[5] * 256 * 256 + data[6] * 256 + data[7] + # self.ssrc = data[8] * 256 * 256 * 256 + data[9] * 256 * 256 + data[10] * 256 + data[11] + + self.payload = data[12:] + + def __repr__(self): + return f"RtpPacket(payload_type={self.payload_type}, marker={self.marker}, sequence_number={self.sequence_number}, timestamp={self.timestamp}, ssrc={self.ssrc})" + + def get_payload(self): + return self.payload + + def get_payload_type(self): + return self.payload_type + + def get_marker(self): + return self.marker + +class RtpJpegPacket(RtpPacket): + def __init__(self, data): + super().__init__(data) + rtp_payload = self.get_payload() + self.type_specific, self.frag_offset, self.frag_type, self.q, self.width, self.height = struct.unpack('>B3s BBBB', rtp_payload[:8]) + self.frag_offset = int.from_bytes(self.frag_offset, byteorder='big') + self.q = int(self.q) + self.width = int(self.width) * 8 + self.height = int(self.height) * 8 + + self.jpeg_data = rtp_payload[8:] + + if 128 <= self.q <= 255: + # bytes 8,9,10 are all 0 based on CStreamer.cpp lines 109-111 + num_quant_bytes = rtp_payload[11] + quant_size = 64 + expected_quant_bytes = 2 * quant_size + if num_quant_bytes != expected_quant_bytes: + print(f"Unexpected quant bytes: {num_quant_bytes}, expected {expected_quant_bytes}") + else: + q0_offset = 12 + q1_offset = q0_offset + quant_size + q1_end = q1_offset + quant_size + self.q0 = rtp_payload[q0_offset:q1_offset] + self.q1 = rtp_payload[q1_offset:q1_end] + self.jpeg_data = rtp_payload[q1_end:] + + def __repr__(self): + return f"RtpJpegPacket(payload_type={self.payload_type}, marker={self.marker}, sequence_number={self.sequence_number}, timestamp={self.timestamp}, ssrc={self.ssrc}, width={self.width}, height={self.height}, q={self.q}, frag_type={self.frag_type}, frag_offset={self.frag_offset})" + + def get_width(self): + return self.width + + def get_frag_offset(self): + return self.frag_offset + + def get_frag_type(self): + return self.frag_type + + def get_height(self): + return self.height + + def get_q0(self): + return self.q0 + + def get_q1(self): + return self.q1 + + def get_jpeg_data(self): + return self.jpeg_data + + +class JpegHeader: + def __init__(self, width, height, q0_quantization_table, q1_quantization_table): + self.width = width + self.height = height + + self.data = io.BytesIO() + self.data.write(b'\xFF\xD8') # Start Of Image (SOI) marker + # JFIF APP0 marker + jfif_app0_marker = bytearray([ + 0xFF, 0xE0, # APP0 marker + 0x00, 0x10, # Length (16 bytes) + 0x4A, 0x46, 0x49, 0x46, 0x00, # JFIF identifier + 0x01, 0x02, # JFIF version 1.2 + 0x01, # Units: DPI + 0x00, 0x48, # Xdensity: 72 DPI + 0x00, 0x48, # Ydensity: 72 DPI + 0x00, 0x00 # No thumbnail (width 0, height 0) + ]) + self.data.write(jfif_app0_marker) + + # Quantization table (DQT) marker for luminance + # marker(0xFFDB), size (0x0043 = 67), index (0x00) + self.data.write(b'\xFF\xDB\x00\x43\x00') + self.data.write(bytearray(q0_quantization_table)) + + # Quantization table (DQT) marker for chrominance + # marker(0xFFDB), size (0x0043 = 67), index (0x01) + self.data.write(b'\xFF\xDB\x00\x43\x01') + self.data.write(bytearray(q1_quantization_table)) + + # Frame header (SOF0) marker + sof0_marker = bytearray([ + 0xFF, 0xC0, # SOF0 marker + 0x00, 0x11, # Length (17 bytes) + 0x08, # Data precision: 8 bits + *self.height.to_bytes(2, 'big'), # 0x01, 0xE0, # Image height: 240 + *self.width.to_bytes(2, 'big'), # 0x01, 0xE0, # Image width: 240 + 0x03, # Number of components: 3 (YCbCr) + 0x01, 0x21, 0x00, # Component 1 (Y): horizontal sampling factor = 2, vertical sampling factor = 1, quantization table ID = 0 + 0x02, 0x11, 0x01, # Component 2 (Cb): horizontal sampling factor = 1, vertical sampling factor = 1, quantization table ID = 1 + 0x03, 0x11, 0x01 # Component 3 (Cr): horizontal sampling factor = 1, vertical sampling factor = 1, quantization table ID = 1 + ]) + self.data.write(sof0_marker) + + self.data.write(bytes(huffman_table)) + + # Scan header (SOS) marker + # marker(0xFFDA), size of SOS (0x000C), num components(0x03), + # component specification parameters, + # spectral selection (0x003F), + # successive appromiation parameters (0x00) + self.data.write(b'\xFF\xDA\x00\x0C\x03\x01\x00\x02\x11\x03\x11\x00\x3F\x00') + + def __repr__(self): + return f"JpegHeader(width={self.width}, height={self.height})" + + def get_data(self): + return self.data.getvalue() + +class JpegFrame: + def __init__(self, RtpJpegPacket): + self.jpeg_header = JpegHeader(RtpJpegPacket.get_width(), RtpJpegPacket.get_height(), RtpJpegPacket.get_q0(), RtpJpegPacket.get_q1()) + self.jpeg_data = RtpJpegPacket.get_jpeg_data() + + def __repr__(self): + return f"JpegFrame(width={self.jpeg_header.width}, height={self.jpeg_header.height}, jpeg_data={self.jpeg_data})" + + def add_packet(self, rtp_jpeg_packet): + self.jpeg_data += rtp_jpeg_packet.get_jpeg_data() + + def get_data(self): + data = io.BytesIO() + data.write(self.jpeg_header.get_data()) + data.write(self.jpeg_data) + data.write(b'\xFF\xD9') # End Of Image (EOI) marker + return data.getvalue() + + class RtspClient: def __init__(self, server, port): self.server = server @@ -146,132 +311,40 @@ def start_receiving_video_stream(self, rtp_port, rtcp_port): def handle_rtp_packet(self): # for this example we'll show the received video stream in an opencv # window - buf = bytearray() cv2.namedWindow('MJPEG Stream', cv2.WINDOW_NORMAL) - + jpeg_frame = None while True: # Process RTP packet in rtp_data rtp_data, addr = self.rtp_socket.recvfrom(8192) - print(f"received rtp packet, len={len(rtp_data)}") - rtp_header = rtp_data[:12] - rtp_payload = rtp_data[12:] # NAL unit - - - version, payload_and_marker, seq, timestamp, ssrc = struct.unpack('>BBHII', rtp_header) - payload_type = (payload_and_marker & 0x7F) - marker_bit = (payload_and_marker & 0b10000000) >> 7 - - # print("\tRTP Header:", [hex(x) for x in list(rtp_header)]) - print(f"\tMarker: {marker_bit}, Payload Type: {payload_type}") - # print(f"Sequence number: {seq}, Timestamp: {timestamp}, SSRC: {ssrc}") - - jpeg_header = rtp_payload[:8] - type_specific, frag_offset, frag_type, q, width, height = struct.unpack('>B3s BBBB', rtp_payload[:8]) - frag_offset = int.from_bytes(frag_offset, byteorder='big') - q = int(q) - width = int(width) * 8 - height = int(height) * 8 - - # print("\tJPEG header:", [hex(x) for x in list(jpeg_header)]) - print(f"\tJpeg image: width={width}, height={height}, type_spec={type_specific}, q={q}, frag_type={frag_type}, frag_offset={frag_offset}") - - jpeg_data = rtp_payload[8:] - - if 64 <= frag_type <= 127: - # there must be a restart marker header - print("\tHave restart header") - restart_header = rtp_payload[8:12] - restart_interval = int((restart_header[0] << 8) | restart_header[1]) - f_bit = True if restart_header[2] & 0x80 else False - l_bit = True if restart_header[2] & 0x80 else False - restart_count = int(((restart_header[2] & 0x3F) << 8) | restart_header[3]) - print(f"\tRestart interval={restart_interval}, f={f_bit}, l={l_bit}, restart_count={restart_count}") - jpeg_data = rtp_payload[12:] - - if 128 <= q <= 255: - print("\tGetting quantization table header") - # bytes 8,9,10 are all 0 based on CStreamer.cpp lines 109-111 - num_quant_bytes = rtp_payload[11] - quant_size = 64 - expected_quant_bytes = 2 * quant_size - if num_quant_bytes != expected_quant_bytes: - print(f"Unexpected quant bytes: {num_quant_bytes}, expected {expected_quant_bytes}") - else: - q0_offset = 12 - q1_offset = q0_offset + quant_size - q1_end = q1_offset + quant_size - q0 = rtp_payload[q0_offset:q1_offset] - q1 = rtp_payload[q1_offset:q1_end] - jpeg_data = rtp_payload[q1_end:] + rtp_packet = RtpJpegPacket(rtp_data) + jpeg_data = rtp_packet.get_jpeg_data() + frag_offset = rtp_packet.get_frag_offset() if frag_offset == 0: - # Create a binary stream to construct the JPEG header - jpeg_header = io.BytesIO() - - # Start Of Image (SOI) marker - jpeg_header.write(b'\xFF\xD8') - - jfif_app0_marker = bytearray([ - 0xFF, 0xE0, # APP0 marker - 0x00, 0x10, # Length (16 bytes) - 0x4A, 0x46, 0x49, 0x46, 0x00, # JFIF identifier - 0x01, 0x02, # JFIF version 1.2 - 0x01, # Units: DPI - 0x00, 0x48, # Xdensity: 72 DPI - 0x00, 0x48, # Ydensity: 72 DPI - 0x00, 0x00 # No thumbnail (width 0, height 0) - ]) - jpeg_header.write(jfif_app0_marker) - - # Quantization table (DQT) marker for luminance - # marker(0xFFDB), size (0x0043 = 67), index (0x00) - jpeg_header.write(b'\xFF\xDB\x00\x43\x00') - jpeg_header.write(bytearray(q0)) - - # Quantization table (DQT) marker for chrominance - # marker(0xFFDB), size (0x0043 = 67), index (0x01) - jpeg_header.write(b'\xFF\xDB\x00\x43\x01') - jpeg_header.write(bytearray(q1)) - - # Frame header (SOF0) marker - sof0_marker = bytearray([ - 0xFF, 0xC0, # SOF0 marker - 0x00, 0x11, # Length (17 bytes) - 0x08, # Data precision: 8 bits - *height.to_bytes(2, 'big'), # 0x01, 0xE0, # Image height: 240 - *width.to_bytes(2, 'big'), # 0x01, 0xE0, # Image width: 240 - 0x03, # Number of components: 3 (YCbCr) - 0x01, 0x21, 0x00, # Component 1 (Y): horizontal sampling factor = 2, vertical sampling factor = 1, quantization table ID = 0 - 0x02, 0x11, 0x01, # Component 2 (Cb): horizontal sampling factor = 1, vertical sampling factor = 1, quantization table ID = 1 - 0x03, 0x11, 0x01 # Component 3 (Cr): horizontal sampling factor = 1, vertical sampling factor = 1, quantization table ID = 1 - ]) - jpeg_header.write(sof0_marker) - - jpeg_header.write(bytes(huffman_table)) - - # Scan header (SOS) marker - # marker(0xFFDA), size of SOS (0x000C), num components(0x03), - # component specification parameters, - # spectral selection (0x003F), - # successive appromiation parameters (0x00) - jpeg_header.write(b'\xFF\xDA\x00\x0C\x03\x01\x00\x02\x11\x03\x11\x00\x3F\x00') - - jpeg_header_bytes = bytearray(jpeg_header.getvalue()) - - print(f"\tAdded header of length {len(jpeg_header_bytes)}") - - buf = jpeg_header_bytes - - # make sure we add the actual jpeg segment data - buf.extend(jpeg_data) - + # this is the first packet of a new frame, so we need to + # create a new JpegFrame object + jpeg_frame = JpegFrame(rtp_packet) + elif jpeg_frame is not None: + # this is a continuation of a previous frame, so we need to + # add the data to the existing JpegFrame object + jpeg_frame.add_packet(rtp_packet) + else: + # we don't have a JpegFrame object yet, so we can't do + # anything with this packet + print(f"Received a packet with frag_offset = {frag_offset} > 0, but no JpegFrame object exists yet") + continue + + # check if this is the last packet of the frame + # (the last packet will have the M bit set) + marker_bit = rtp_packet.get_marker() if marker_bit: - # Add the JPEG end marker (EOI) - buf.extend(b'\xFF\xD9') - print(f"Decoding image size={len(buf)}") + # this is the last packet of the frame, so we can decode + # the frame and show it in the opencv window + buf = jpeg_frame.get_data() + # print(f"Decoding image size={len(buf)}") frame = cv2.imdecode(np.frombuffer(buf, dtype=np.uint8), cv2.IMREAD_COLOR) if frame is not None: - print(f"Decoded frame: {frame.shape}\n\n") + # print(f"Decoded frame: {frame.shape}\n\n") # our images are flipped vertically, fix it :) # 0 = vertical, 1 = horizontal, -1 = both vertical and horiztonal frame = cv2.flip(frame, 0) From ab5d62f524d966bbdecee60b496336a62e338ad1 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Fri, 21 Apr 2023 17:30:41 -0500 Subject: [PATCH 03/11] feat(rtsp_client): working impl * Added c++ versions of python classes from working python example * Updated python example to be refactored into classes * Added example for running the rtsp client on esp32 --- components/rtsp_client/CMakeLists.txt | 3 + components/rtsp_client/example/CMakeLists.txt | 22 + components/rtsp_client/example/README.md | 67 +++ .../rtsp_client/example/main/CMakeLists.txt | 2 + .../example/main/Kconfig.projbuild | 21 + .../example/main/rtsp_client_example.cpp | 108 +++++ components/rtsp_client/example/partitions.csv | 5 + .../rtsp_client/example/sdkconfig.defaults | 26 + components/rtsp_client/include/jpeg_frame.hpp | 93 ++++ .../rtsp_client/include/jpeg_header.hpp | 176 +++++++ .../rtsp_client/include/rtp_jpeg_packet.hpp | 75 +++ components/rtsp_client/include/rtp_packet.hpp | 78 +++ .../rtsp_client/include/rtsp_client.hpp | 454 ++++++++++++++++++ components/rtsp_client/python/rtsp_client.py | 5 +- 14 files changed, 1134 insertions(+), 1 deletion(-) create mode 100644 components/rtsp_client/CMakeLists.txt create mode 100644 components/rtsp_client/example/CMakeLists.txt create mode 100644 components/rtsp_client/example/README.md create mode 100644 components/rtsp_client/example/main/CMakeLists.txt create mode 100644 components/rtsp_client/example/main/Kconfig.projbuild create mode 100644 components/rtsp_client/example/main/rtsp_client_example.cpp create mode 100644 components/rtsp_client/example/partitions.csv create mode 100644 components/rtsp_client/example/sdkconfig.defaults create mode 100644 components/rtsp_client/include/jpeg_frame.hpp create mode 100644 components/rtsp_client/include/jpeg_header.hpp create mode 100644 components/rtsp_client/include/rtp_jpeg_packet.hpp create mode 100644 components/rtsp_client/include/rtp_packet.hpp create mode 100644 components/rtsp_client/include/rtsp_client.hpp diff --git a/components/rtsp_client/CMakeLists.txt b/components/rtsp_client/CMakeLists.txt new file mode 100644 index 0000000..ca706fc --- /dev/null +++ b/components/rtsp_client/CMakeLists.txt @@ -0,0 +1,3 @@ +idf_component_register( + INCLUDE_DIRS "include" + PRIV_REQUIRES logger socket task) diff --git a/components/rtsp_client/example/CMakeLists.txt b/components/rtsp_client/example/CMakeLists.txt new file mode 100644 index 0000000..3bcdead --- /dev/null +++ b/components/rtsp_client/example/CMakeLists.txt @@ -0,0 +1,22 @@ +# The following lines of boilerplate have to be in your project's CMakeLists +# in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.5) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) + +# add the component directories that we want to use +set(EXTRA_COMPONENT_DIRS + "../../../components/" + "../../../components/espp/components" +) + +set( + COMPONENTS + "main esptool_py logger task socket wifi rtsp_client" + CACHE STRING + "List of components to include" + ) + +project(rtsp_client_example) + +set(CMAKE_CXX_STANDARD 20) diff --git a/components/rtsp_client/example/README.md b/components/rtsp_client/example/README.md new file mode 100644 index 0000000..ffe0beb --- /dev/null +++ b/components/rtsp_client/example/README.md @@ -0,0 +1,67 @@ +_Note that this is a template for an ESP-IDF example README.md file. When using this template, replace all these emphasised placeholders with example-specific content._ + +| Supported Targets | _Supported target, e.g. ESP32_ | _Another supported target, e.g. ESP32-S3_ | +| ----------------- | ------------------------------ | ----------------------------------------- | + +_If the example supports all targets supported by ESP-IDF then the table can be omitted_ +# _Example Title_ + +(See the README.md file in the upper level 'examples' directory for more information about examples.) + +_What is this example? What does it do?_ + +_What features of ESP-IDF does it use?_ + +_What could someone create based on this example? ie applications/use cases/etc_ + +_If there are any acronyms or Espressif-only words used here, explain them or mention where in the datasheet/TRM this information can be found._ + +## How to use example + +### Hardware Required + +_If possible, example should be able to run on any commonly available ESP32 development board. Otherwise, describe what specific hardware should be used._ + +_If any other items (server, BLE device, app, second chip, whatever) are needed, mention them here. Include links if applicable. Explain how to set them up._ + +### Configure the project + +``` +idf.py menuconfig +``` + +* _If there is any project configuration that the user must set for this example, mention this here._ + +### Build and Flash + +Build the project and flash it to the board, then run monitor tool to view serial output: + +``` +idf.py -p PORT flash monitor +``` + +(Replace PORT with the name of the serial port to use.) + +(To exit the serial monitor, type ``Ctrl-]``.) + +See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects. + +## Example Output + +_Include an example of the console output from the running example, here:_ + +``` +Use this style for pasting the log. +``` + +_If the user is supposed to interact with the example at this point (read/write GATT attribute, send HTTP request, press button, etc. then mention it here)_ + +_For examples where ESP32 is connected with some other hardware, include a table or schematics with connection details._ + +## Troubleshooting + +_If there are any likely problems or errors which many users might encounter, mention them here. Remove this section for very simple examples where nothing is likely to go wrong._ + +## Example Breakdown + +_If the example source code is lengthy, complex, or cannot be easily understood, use this section to break down and explain the source code. This can be done by breaking down the execution path step by step, or explaining what each major function/task/source file does. Add sub titles if necessary. Remove this section for very simple examples where the source code is self explanatory._ \ No newline at end of file diff --git a/components/rtsp_client/example/main/CMakeLists.txt b/components/rtsp_client/example/main/CMakeLists.txt new file mode 100644 index 0000000..a941e22 --- /dev/null +++ b/components/rtsp_client/example/main/CMakeLists.txt @@ -0,0 +1,2 @@ +idf_component_register(SRC_DIRS "." + INCLUDE_DIRS ".") diff --git a/components/rtsp_client/example/main/Kconfig.projbuild b/components/rtsp_client/example/main/Kconfig.projbuild new file mode 100644 index 0000000..d68cd52 --- /dev/null +++ b/components/rtsp_client/example/main/Kconfig.projbuild @@ -0,0 +1,21 @@ +menu "WiFi Example Configuration" + + config ESP_WIFI_SSID + string "WiFi SSID" + default "myssid" + help + SSID (network name) for the example to connect to. + + config ESP_WIFI_PASSWORD + string "WiFi Password" + default "mypassword" + help + WiFi password (WPA or WPA2) for the example to use. + + config ESP_MAXIMUM_RETRY + int "Maximum retry" + default 5 + help + Set the Maximum retry to avoid station reconnecting to the AP unlimited when the AP is really inexistent. + +endmenu diff --git a/components/rtsp_client/example/main/rtsp_client_example.cpp b/components/rtsp_client/example/main/rtsp_client_example.cpp new file mode 100644 index 0000000..15742d8 --- /dev/null +++ b/components/rtsp_client/example/main/rtsp_client_example.cpp @@ -0,0 +1,108 @@ +#include +#include +#include +#include +#include + +#if CONFIG_ESP32_WIFI_NVS_ENABLED +#include "nvs_flash.h" +#endif + +#include "logger.hpp" +#include "task.hpp" +#include "tcp_socket.hpp" +#include "udp_socket.hpp" +#include "wifi_sta.hpp" + +#include "jpeg_frame.hpp" +#include "rtsp_client.hpp" + +using namespace std::chrono_literals; +using namespace std::placeholders; + +extern "C" void app_main(void) { + espp::Logger logger({.tag = "RtspClient example", .level = espp::Logger::Verbosity::INFO}); + logger.info("Starting RtspClient example"); + +#if CONFIG_ESP32_WIFI_NVS_ENABLED + // Initialize NVS + esp_err_t ret = nvs_flash_init(); + if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) { + ESP_ERROR_CHECK(nvs_flash_erase()); + ret = nvs_flash_init(); + } + ESP_ERROR_CHECK(ret); +#endif + + // create a wifi station here so that LwIP will be init for this example + espp::WifiSta wifi_sta({ + .ssid = CONFIG_ESP_WIFI_SSID, + .password = CONFIG_ESP_WIFI_PASSWORD, + .num_connect_retries = CONFIG_ESP_MAXIMUM_RETRY, + .on_connected = nullptr, + .on_disconnected = nullptr, + .on_got_ip = [](ip_event_got_ip_t* eventdata) { + fmt::print("got IP: {}.{}.{}.{}\n", IP2STR(&eventdata->ip_info.ip)); + } + }); + + // wait until wifi is connected + while (!wifi_sta.is_connected()) { + std::this_thread::sleep_for(1s); + } + + std::error_code ec; + espp::RtspClient rtsp_client({ + .server_address = "192.168.86.181", + .rtsp_port = 8554, + .path = "/mjpeg/1", + .on_jpeg_frame = [](std::unique_ptr jpeg_frame) { + auto jpeg_data = jpeg_frame->get_data(); + auto jpeg_size = jpeg_data.size(); + fmt::print("Got JPEG frame: {} bytes\n", jpeg_size); + }, + .log_level = espp::Logger::Verbosity::INFO, + }); + + rtsp_client.connect(ec); + if (ec) { + logger.error("Failed to connect to RTSP server: {}", ec.message()); + return; + } + + rtsp_client.describe(ec); + if (ec) { + logger.error("Failed to describe stream: {}", ec.message()); + return; + } + + rtsp_client.setup(ec); + if (ec) { + logger.error("Failed to setup stream: {}", ec.message()); + return; + } + + rtsp_client.play(ec); + if (ec) { + logger.error("Failed to play stream: {}", ec.message()); + return; + } + + // sleep for 5 seconds + std::this_thread::sleep_for(5s); + + // NOTE: the current server doesn't properly respond to the teardown request, so we'll just + // ignore the error here + rtsp_client.teardown(ec); + + // NOTE: the current server doesn't properly respond to the teardown request, so we'll just + // ignore the error here + rtsp_client.disconnect(ec); + + logger.info("RtspClient example finished"); + + // wait forever + while (true) { + std::this_thread::sleep_for(1s); + } +} diff --git a/components/rtsp_client/example/partitions.csv b/components/rtsp_client/example/partitions.csv new file mode 100644 index 0000000..c4217ab --- /dev/null +++ b/components/rtsp_client/example/partitions.csv @@ -0,0 +1,5 @@ +# ESP-IDF Partition Table +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x6000, +phy_init, data, phy, 0xf000, 0x1000, +factory, app, factory, 0x10000, 1500K, diff --git a/components/rtsp_client/example/sdkconfig.defaults b/components/rtsp_client/example/sdkconfig.defaults new file mode 100644 index 0000000..397db0b --- /dev/null +++ b/components/rtsp_client/example/sdkconfig.defaults @@ -0,0 +1,26 @@ +CONFIG_IDF_TARGET="esp32s3" + +CONFIG_COMPILER_OPTIMIZATION_PERF=y +# CONFIG_COMPILER_OPTIMIZATION_SIZE=y + +CONFIG_FREERTOS_HZ=1000 + +CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y +CONFIG_ESPTOOLPY_FLASHSIZE="16MB" + +# +# Common ESP-related +# +CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=4096 +CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192 + +# SPIRAM Configuration +CONFIG_SPIRAM=y +CONFIG_SPIRAM_USE=y +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y + +# ESP32-specific +# +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ=240 diff --git a/components/rtsp_client/include/jpeg_frame.hpp b/components/rtsp_client/include/jpeg_frame.hpp new file mode 100644 index 0000000..740969b --- /dev/null +++ b/components/rtsp_client/include/jpeg_frame.hpp @@ -0,0 +1,93 @@ +#pragma once + +#include "rtp_jpeg_packet.hpp" +#include "jpeg_header.hpp" + +namespace espp { + /// A class that represents a complete JPEG frame. + /// + /// This class is used to collect the JPEG scans that are received in RTP + /// packets and to serialize them into a complete JPEG frame. + class JpegFrame { + public: + + /// Construct a JpegFrame from a RtpJpegPacket. + /// + /// This constructor will parse the header of the packet and add the JPEG + /// data to the frame. + /// + /// @param packet The packet to parse. + explicit JpegFrame(const RtpJpegPacket& packet) { + // parse the header + header_ = std::make_unique(packet.get_width(), packet.get_height(), packet.get_q_table(0), packet.get_q_table(1)); + // add the jpeg data + scans_.push_back(packet.get_jpeg_data()); + } + + /// Append a RtpJpegPacket to the frame. + /// This will add the JPEG data to the frame. + /// @param packet The packet containing the scan to append. + void append(const RtpJpegPacket& packet) { + add_scan(packet); + } + + /// Append a JPEG scan to the frame. + /// This will add the JPEG data to the frame. + /// @param packet The packet containing the scan to append. + void add_scan(const RtpJpegPacket& packet) { + add_scan(packet.get_jpeg_data()); + } + + /// Append a JPEG scan to the frame. + /// This will add the JPEG data to the frame. + /// @param scan The jpeg scan to append. + void add_scan(std::string_view scan) { + scans_.push_back(scan); + } + + /// Serialize the frame. + /// + /// This will serialize the header and all scans into a single buffer which + /// can be sent over the network. You can get the serialized data using + /// get_data(). + /// + /// @note This method must be called before get_data() can be used. + /// @note This method should only be called once. + void serialize() { + auto header_data = header_->get_data(); + auto scan_bytes = 0; + for (auto& scan : scans_) { + scan_bytes += scan.size(); + } + data_.resize(header_data.size() + scan_bytes + 2);; + + int offset = 0; + // serialize the header + memcpy(data_.data(), header_data.data(), header_data.size()); + offset += header_data.size(); + // serialize the scans + for (auto& scan : scans_) { + memcpy(data_.data() + offset, scan.data(), scan.size()); + offset += scan.size(); + } + // add the EOI marker + data_[offset++] = 0xFF; + data_[offset++] = 0xD9; + } + + /// Get the serialized data. + /// + /// This will return the serialized data. You must call serialize() before + /// calling this method. + /// + /// @return The serialized data. + std::string_view get_data() const { + return std::string_view(data_.data(), data_.size()); + } + + protected: + std::unique_ptr header_; + std::vector scans_; + std::vector data_; + }; +} // namespace espp diff --git a/components/rtsp_client/include/jpeg_header.hpp b/components/rtsp_client/include/jpeg_header.hpp new file mode 100644 index 0000000..f468fac --- /dev/null +++ b/components/rtsp_client/include/jpeg_header.hpp @@ -0,0 +1,176 @@ +# pragma once + +#include +#include +#include + +namespace espp { + /// A class to generate a JPEG header for a given image size and quantization tables. + /// The header is generated once and then cached for future use. + /// The header is generated according to the JPEG standard and is compatible with + /// the ESP32 camera driver. + class JpegHeader { + public: + /// Create a JPEG header for a given image size and quantization tables. + /// @param width The image width in pixels. + /// @param height The image height in pixels. + /// @param q0_table The quantization table for the Y channel. + /// @param q1_table The quantization table for the Cb and Cr channels. + explicit JpegHeader(int width, int height, std::string_view q0_table, std::string_view q1_table) + : width_(width), height_(height), q0_table_(q0_table), q1_table_(q1_table) { + serialize(); + } + + ~JpegHeader() {} + + /// Get the JPEG header data. + /// @return The JPEG header data. + std::string_view get_data() { + return std::string_view(data_.data(), data_.size()); + } + + protected: + + static constexpr int SOF0_SIZE = 19; + static constexpr int DQT_HEADER_SIZE = 4; + + // JFIF APP0 Marker for version 1.2 with 72 DPI and no thumbnail + static constexpr char JFIF_APP0_DATA[] = { + 0xFF, 0xE0, // APP0 marker + 0x00, 0x10, // Length of APP0 data (16 bytes) + 0x4A, 0x46, 0x49, 0x46, 0x00, // Identifier: ASCII "JFIF\0" + 0x01, 0x02, // Version number (1.2) + 0x01, // Units: 1 = dots per inch + 0x00, 0x48, // X density (72 DPI) + 0x00, 0x48, // Y density (72 DPI) + 0x00, 0x00 // No thumbnail + }; + + static constexpr char HUFFMAN_TABLES[] = { + // 1st table + // Default luminance DC Huffman table + 0xff, 0xc4, 0x00, 0x1f, 0x00, // header + 0x00, 0x01, 0x05, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x02, 0x03, + 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, + // 2nd table + // Default luminance AC Huffman table + 0xff, 0xc4, 0x00, 0xb5, 0x10, // header + 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01, 0x7d, 0x01, 0x02, + 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, + 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, + 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, + 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, + 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, + 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, + 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2, 0xe3, 0xe4, + 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, + }; + + // Scan header (SOS) + static constexpr char SOS[] = { + 0xFF, 0xDA, // SOS marker + 0x00, 0x0C, // length + 0x03, // number of components + 0x01, 0x00, 0x02, 0x11, 0x03, 0x11, 0x00, 0x3F, 0x00 // components + }; + + + int add_sof0(int offset) { + // add the SOF0 marker + data_[offset++] = 0xFF; + data_[offset++] = 0xC0; + // add the length of the marker + data_[offset++] = 0x00; + data_[offset++] = 0x11; + // add the precision + data_[offset++] = 0x08; + // add the height + data_[offset++] = (height_ >> 8) & 0xFF; + data_[offset++] = height_ & 0xFF; + // add the width + data_[offset++] = (width_ >> 8) & 0xFF; + data_[offset++] = width_ & 0xFF; + // add the number of components + data_[offset++] = 0x03; + // add the Y component + data_[offset++] = 0x01; + data_[offset++] = 0x21; + data_[offset++] = 0x00; + // add the Cb component + data_[offset++] = 0x02; + data_[offset++] = 0x11; + data_[offset++] = 0x01; + // add the Cr component + data_[offset++] = 0x03; + data_[offset++] = 0x11; + data_[offset++] = 0x01; + return offset; + } + + void serialize() { + int header_size = + 2 + + sizeof(JFIF_APP0_DATA) + + DQT_HEADER_SIZE + + q0_table_.size() + + DQT_HEADER_SIZE + + q1_table_.size() + + SOF0_SIZE + + sizeof(HUFFMAN_TABLES) + + sizeof(SOS); + // serialize the jpeg header to the data_ vector + data_.resize(header_size); + int offset = 0; + + // add the SOI marker + data_[offset++] = 0xFF; + data_[offset++] = 0xD8; + + // add the JFIF APP0 marker + memcpy(data_.data() + offset, JFIF_APP0_DATA, sizeof(JFIF_APP0_DATA)); + offset += sizeof(JFIF_APP0_DATA); + + // add the DQT marker for luminance + data_[offset++] = 0xFF; + data_[offset++] = 0xDB; + data_[offset++] = 0x00; + data_[offset++] = 0x43; + data_[offset++] = 0x00; + memcpy(data_.data() + offset, q0_table_.data(), q0_table_.size()); + offset += q0_table_.size(); + + // add the DQT marker for chrominance + data_[offset++] = 0xFF; + data_[offset++] = 0xDB; + data_[offset++] = 0x00; + data_[offset++] = 0x43; + data_[offset++] = 0x01; + memcpy(data_.data() + offset, q1_table_.data(), q1_table_.size()); + offset += q1_table_.size(); + + // add the SOF0 + offset = add_sof0(offset); + + // add huffman tables + memcpy(data_.data() + offset, HUFFMAN_TABLES, sizeof(HUFFMAN_TABLES)); + offset += sizeof(HUFFMAN_TABLES); + + // add the SOS marker + memcpy(data_.data() + offset, SOS, sizeof(SOS)); + offset += sizeof(SOS); + } + + int width_; + int height_; + std::string_view q0_table_; + std::string_view q1_table_; + + std::vector data_; + }; +} // namespace espp diff --git a/components/rtsp_client/include/rtp_jpeg_packet.hpp b/components/rtsp_client/include/rtp_jpeg_packet.hpp new file mode 100644 index 0000000..5f37f52 --- /dev/null +++ b/components/rtsp_client/include/rtp_jpeg_packet.hpp @@ -0,0 +1,75 @@ +#pragma once + +#include "rtp_packet.hpp" + +namespace espp { + class RtpJpegPacket : public RtpPacket { + public: + explicit RtpJpegPacket(std::string_view data) : RtpPacket(data) { + parse_mjpeg_header(); + } + + ~RtpJpegPacket() {} + + int get_type_specific() const { return type_specific_; } + int get_offset() const { return offset_; } + int get_q() const { return q_; } + int get_width() const { return width_; } + int get_height() const { return height_; } + + int get_mjpeg_header_size() const { return MJPEG_HEADER_SIZE; } + std::string_view get_mjpeg_header() { + return std::string_view(packet_.data() + RTP_HEADER_SIZE, MJPEG_HEADER_SIZE); + } + + int get_q_table_size() const { return Q_TABLE_SIZE; } + int get_num_q_tables() const { return q_table_indices_.size(); } + std::string_view get_q_table(int index) const { + if (index < get_num_q_tables()) { + return std::string_view(packet_.data() + q_table_indices_[index], Q_TABLE_SIZE); + } + return {}; + } + + std::string_view get_jpeg_data() const { + int quant_size = get_num_q_tables() * Q_TABLE_SIZE; + return std::string_view(packet_.data() + RTP_HEADER_SIZE + MJPEG_HEADER_SIZE + quant_size, + packet_.size() - RTP_HEADER_SIZE - MJPEG_HEADER_SIZE - quant_size); + } + + + protected: + static constexpr int MJPEG_HEADER_SIZE = 8; + static constexpr int NUM_Q_TABLES = 2; + static constexpr int Q_TABLE_SIZE = 64; + + void parse_mjpeg_header() { + int offset = RTP_HEADER_SIZE + MJPEG_HEADER_SIZE; + type_specific_ = packet_[offset]; + offset_ = (packet_[offset + 1] << 8) | packet_[offset + 2]; + q_ = packet_[offset + 3]; + width_ = (packet_[offset + 4] << 8) | packet_[offset + 5]; + height_ = (packet_[offset + 6] << 8) | packet_[offset + 7]; + + // If the Q value is between 128 and 256, then the packet contains + // quantization tables. + if (128 <= q_ && q_ <= 256) { + int num_quant_bytes = packet_[offset + 11]; + int expected_num_quant_bytes = NUM_Q_TABLES * Q_TABLE_SIZE; + if (num_quant_bytes == expected_num_quant_bytes) { + q_table_indices_.resize(NUM_Q_TABLES); + for (int i = 0; i < NUM_Q_TABLES; i++) { + q_table_indices_[i] = offset + 12 + (i * Q_TABLE_SIZE); + } + } + } + } + + int type_specific_; + int offset_; + int q_; + int width_; + int height_; + std::vector q_table_indices_; + }; +} // namespace espp diff --git a/components/rtsp_client/include/rtp_packet.hpp b/components/rtsp_client/include/rtp_packet.hpp new file mode 100644 index 0000000..707b5d5 --- /dev/null +++ b/components/rtsp_client/include/rtp_packet.hpp @@ -0,0 +1,78 @@ +#pragma once + +#include +#include +#include + +namespace espp { + /// RtpPacket is a class to parse RTP packet. + class RtpPacket { + public: + /// Construct an RtpPacket from a string_view. + /// Store the string_view in the packet_ vector and parses the header. + /// @param data The string_view to parse. + explicit RtpPacket(std::string_view data) { + packet_.assign(data.begin(), data.end()); + parse_rtp_header(); + } + + ~RtpPacket() {} + + /// Getters for the RTP header fields. + int get_version() const { return version_; } + bool get_padding() const { return padding_; } + bool get_extension() const { return extension_; } + int get_csrc_count() const { return csrc_count_; } + bool get_marker() const { return marker_; } + int get_payload_type() const { return payload_type_; } + int get_sequence_number() const { return sequence_number_; } + int get_timestamp() const { return timestamp_; } + int get_ssrc() const { return ssrc_; } + + /// Get a string_view of the whole packet. + /// @return A string_view of the whole packet. + std::string_view get_packet() { + return std::string_view(packet_.data(), packet_.size()); + } + + /// Get a string_view of the RTP header. + /// @return A string_view of the RTP header. + std::string_view get_packet_header() { + return std::string_view(packet_.data(), RTP_HEADER_SIZE); + } + + /// Get a string_view of the payload. + /// @return A string_view of the payload. + std::string_view get_payload() { + return std::string_view(packet_.data() + RTP_HEADER_SIZE, payload_size_); + } + + protected: + static constexpr int RTP_HEADER_SIZE = 12; + + void parse_rtp_header() { + version_ = (packet_[0] & 0xC0) >> 6; + padding_ = (packet_[0] & 0x20) >> 5; + extension_ = (packet_[0] & 0x10) >> 4; + csrc_count_ = packet_[0] & 0x0F; + marker_ = (packet_[1] & 0x80) >> 7; + payload_type_ = packet_[1] & 0x7F; + sequence_number_ = (packet_[2] << 8) | packet_[3]; + timestamp_ = (packet_[4] << 24) | (packet_[5] << 16) | (packet_[6] << 8) | packet_[7]; + ssrc_ = (packet_[8] << 24) | (packet_[9] << 16) | (packet_[10] << 8) | packet_[11]; + payload_size_ = packet_.size() - RTP_HEADER_SIZE; + } + + std::vector packet_; + int version_; + bool padding_; + bool extension_; + int csrc_count_; + bool marker_; + int payload_type_; + int sequence_number_; + int timestamp_; + int ssrc_; + int payload_size_; + }; +} // namespace espp diff --git a/components/rtsp_client/include/rtsp_client.hpp b/components/rtsp_client/include/rtsp_client.hpp new file mode 100644 index 0000000..a1d5eaa --- /dev/null +++ b/components/rtsp_client/include/rtsp_client.hpp @@ -0,0 +1,454 @@ +#pragma once + +#include +#include +#include +#include + +#include "logger.hpp" +#include "tcp_socket.hpp" +#include "udp_socket.hpp" + +#include "jpeg_frame.hpp" + +namespace espp { + + /// A class for interacting with an RTSP server using RTP and RTCP over UDP + class RtspClient { + public: + using jpeg_frame_callback_t = std::function jpeg_frame)>; + + /// Configuration for the RTSP client + struct Config { + std::string server_address; ///< The server IP Address to connect to + int rtsp_port{8554}; ///< The port of the RTSP server + std::string path{"/mjpeg/1"}; ///< The path to the RTSP stream on the server. Will be appended to the server address and port to form the full path of the form "rtsp://:" + jpeg_frame_callback_t on_jpeg_frame; ///< The callback to call when a JPEG frame is received + espp::Logger::Verbosity log_level = espp::Logger::Verbosity::INFO; ///< The verbosity of the logger + }; + + /// Constructor + /// \param config The configuration for the RTSP client + explicit RtspClient(const Config& config) + : server_address_(config.server_address), + rtsp_port_(config.rtsp_port), + rtsp_socket_({.log_level = espp::Logger::Verbosity::WARN}), + rtp_socket_({.log_level = espp::Logger::Verbosity::WARN}), + rtcp_socket_({.log_level = espp::Logger::Verbosity::WARN}), + on_jpeg_frame_(config.on_jpeg_frame), + cseq_(0), + path_("rtsp://" + server_address_ + ":" + std::to_string(rtsp_port_) + config.path), + logger_({.tag = "RtspClient", .level = config.log_level}) { + } + + /// Destructor + /// Disconnects from the RTSP server + ~RtspClient() { + std::error_code ec; + disconnect(ec); + if (ec) { + logger_.error("Error disconnecting: {}", ec.message()); + } + } + + /// Send an RTSP request to the server + /// \note This is a blocking call + /// \note This will parse the response and set the session ID if it is + /// present in the response. If the response is not a 200 OK, then + /// an error code will be set and the response will be returned. + /// If the response is a 200 OK, then the response will be returned + /// and the error code will be set to success. + /// \param method The method to use for connecting. + /// Options are "OPTIONS", "DESCRIBE", "SETUP", "PLAY", and "TEARDOWN" + /// \param path The path to the RTSP stream on the server. + /// \param extra_headers Any extra headers to send with the request. These + /// will be added to the request after the CSeq and Session headers. The + /// key is the header name and the value is the header value. For example, + /// {"Accept": "application/sdp"} will add "Accept: application/sdp" to the + /// request. The "User-Agent" header will be added automatically. The + /// "CSeq" and "Session" headers will be added automatically. + /// The "Accept" header will be added automatically. The "Transport" + /// header will be added automatically for the "SETUP" method. Defaults to + /// an empty map. + /// \param ec The error code to set if an error occurs + /// \return The response from the server + std::string send_request(const std::string& method, const std::string& path, const std::unordered_map& extra_headers, std::error_code& ec) { + // send the request + std::string request = method + " " + path + " RTSP/1.0\r\n"; + request += "CSeq: " + std::to_string(cseq_) + "\r\n"; + if (session_id_.size() > 0) { + request += "Session: " + session_id_ + "\r\n"; + } + for (auto& [key, value] : extra_headers) { + request += key + ": " + value + "\r\n"; + } + request += "User-Agent: rtsp-client\r\n"; + request += "Accept: application/sdp\r\n"; + request += "\r\n"; + std::string response; + auto transmit_config = espp::TcpSocket::TransmitConfig{ + .wait_for_response = true, + .response_size = 1024, + .on_response_callback = [&response](auto &response_vector) { response.assign(response_vector.begin(), response_vector.end()); }, + }; + // NOTE: now this call blocks until the response is received + logger_.debug("Request:\n{}", request); + if (!rtsp_socket_.transmit(request, transmit_config)) { + ec = std::make_error_code(std::errc::io_error); + logger_.error("Failed to send request"); + return {}; + } + + // TODO: how to keep receiving until we get the full response? + // if (response.find("\r\n\r\n") != std::string::npos) { + // break; + // } + + // parse the response + logger_.debug("Response:\n{}", response); + if (parse_response(response, ec)) { + return response; + } + return {}; + } + + /// Connect to the RTSP server + /// Connects to the RTSP server and sends the OPTIONS request. + /// \param ec The error code to set if an error occurs + void connect(std::error_code& ec) { + // exit early if error code is already set + if (ec) { + return; + } + + auto did_connect = rtsp_socket_.connect({ + .ip_address = server_address_, + .port = static_cast(rtsp_port_), + }); + if (!did_connect) { + ec = std::make_error_code(std::errc::io_error); + logger_.error("Failed to connect to {}:{}", server_address_, rtsp_port_); + return; + } + + // send the options request + send_request("OPTIONS", "*", {}, ec); + } + + /// Disconnect from the RTSP server + /// Disconnects from the RTSP server and sends the TEARDOWN request. + /// \param ec The error code to set if an error occurs + void disconnect(std::error_code& ec) { + // send the teardown request + teardown(ec); + rtsp_socket_.reinit(); + } + + /// Describe the RTSP stream + /// Sends the DESCRIBE request to the RTSP server and parses the response. + /// \param ec The error code to set if an error occurs + void describe(std::error_code& ec) { + // exit early if the error code is set + if (ec) { + return; + } + // send the describe request + auto response = send_request("DESCRIBE", path_, {}, ec); + if (ec) { + return; + } + // sdp response is of the form: + // std::regex sdp_regex("m=video (\\d+) RTP/AVP (\\d+)"); + // parse the sdp response and get the video port without using regex + // this is a very simple sdp parser that only works for this specific case + auto sdp_start = response.find("m=video"); + if (sdp_start == std::string::npos) { + ec = std::make_error_code(std::errc::wrong_protocol_type); + logger_.error("Invalid sdp"); + return; + } + auto sdp_end = response.find("\r\n", sdp_start); + if (sdp_end == std::string::npos) { + ec = std::make_error_code(std::errc::protocol_error); + logger_.error("Incomplete sdp"); + return; + } + auto sdp = response.substr(sdp_start, sdp_end - sdp_start); + auto port_start = sdp.find(" "); + if (port_start == std::string::npos) { + ec = std::make_error_code(std::errc::protocol_error); + logger_.error("Could not find port start"); + return; + } + auto port_end = sdp.find(" ", port_start + 1); + if (port_end == std::string::npos) { + ec = std::make_error_code(std::errc::protocol_error); + logger_.error("Could not find port end"); + return; + } + auto port = sdp.substr(port_start + 1, port_end - port_start - 1); + video_port_ = std::stoi(port); + logger_.debug("Video port: {}", video_port_); + auto payload_type_start = sdp.find(" ", port_end + 1); + if (payload_type_start == std::string::npos) { + ec = std::make_error_code(std::errc::protocol_error); + logger_.error("Could not find payload type start"); + return; + } + auto payload_type = sdp.substr(payload_type_start + 1, sdp.size() - payload_type_start - 1); + video_payload_type_ = std::stoi(payload_type); + logger_.debug("Video payload type: {}", video_payload_type_); + } + + /// Setup the RTSP stream + /// \note Starts the RTP and RTCP threads. + /// Sends the SETUP request to the RTSP server and parses the response. + /// \note The default ports are 5000 and 5001 for RTP and RTCP respectively. + /// \param ec The error code to set if an error occurs + void setup(std::error_code& ec) { + // default to rtp and rtcp client ports 5000 and 5001 + setup(5000, 50001, ec); + } + + /// Setup the RTSP stream + /// Sends the SETUP request to the RTSP server and parses the response. + /// \note Starts the RTP and RTCP threads. + /// \param rtp_port The RTP client port + /// \param rtcp_port The RTCP client port + /// \param ec The error code to set if an error occurs + void setup(size_t rtp_port, size_t rtcp_port, std::error_code& ec) { + // exit early if the error code is set + if (ec) { + return; + } + + // set up the transport header with the rtp and rtcp ports + auto transport_header = + "RTP/AVP;unicast;client_port=" + + std::to_string(rtp_port) + "-" + std::to_string(rtcp_port); + + // send the setup request + auto response = send_request("SETUP", path_, {{"Transport", transport_header}}, ec); + if (ec) { + return; + } + + init_rtp(rtp_port, ec); + init_rtcp(rtcp_port, ec); + } + + /// Play the RTSP stream + /// Sends the PLAY request to the RTSP server and parses the response. + /// \param ec The error code to set if an error occurs + void play(std::error_code& ec) { + // exit early if the error code is set + if (ec) { + return; + } + // send the play request + send_request("PLAY", path_ , {}, ec); + } + + /// Pause the RTSP stream + /// Sends the PAUSE request to the RTSP server and parses the response. + /// \param ec The error code to set if an error occurs + void pause(std::error_code& ec) { + // exit early if the error code is set + if (ec) { + return; + } + // send the pause request + send_request("PAUSE", path_, {}, ec); + } + + /// Teardown the RTSP stream + /// Sends the TEARDOWN request to the RTSP server and parses the response. + /// \param ec The error code to set if an error occurs + void teardown(std::error_code& ec) { + // exit early if the error code is set + if (ec) { + return; + } + // send the teardown request + send_request("TEARDOWN", path_, {}, ec); + } + + protected: + /// Parse the RTSP response + /// \note Parses response data for the following fields: + /// - Status code + /// - Status message + /// - Session + /// \note Increments the sequence number on success. + /// \param response_data The response data to parse + /// \param ec The error code to set if an error occurs + /// \return True if the response was parsed successfully, false otherwise + bool parse_response(const std::string& response_data, std::error_code& ec) { + // exit early if the error code is set + if (ec) { + return false; + } + if (response_data.empty()) { + ec = std::make_error_code(std::errc::no_message); + logger_.error("Empty response"); + return false; + } + // RTP response is of the form: + // std::regex response_regex("RTSP/1.0 (\\d+) (.*)\r\n(.*)\r\n\r\n"); + // parse the response but don't use regex since it may be slow on embedded platforms + // make sure it matches the expected response format + if (response_data.find("RTSP/1.0") != 0) { + ec = std::make_error_code(std::errc::protocol_error); + logger_.error("Invalid response"); + return false; + } + // parse the status code and message + int status_code = std::stoi(response_data.substr(9, 3)); + std::string status_message = response_data.substr(13, response_data.find("\r\n") - 13); + if (status_code != 200) { + ec = std::make_error_code(std::errc::protocol_error); + logger_.error(std::string("Request failed: ") + status_message); + return false; + } + // parse the session id + auto session_pos = response_data.find("Session: "); + if (session_pos != std::string::npos) { + session_id_ = response_data.substr(session_pos + 9, response_data.find("\r\n", session_pos) - session_pos - 9); + } + // increment the cseq + cseq_++; + return true; + } + + /// Initialize the RTP socket + /// \note Starts the RTP socket task. + /// \param rtp_port The RTP client port + /// \param ec The error code to set if an error occurs + void init_rtp(size_t rtp_port, std::error_code& ec) { + // exit early if the error code is set + if (ec) { + return; + } + logger_.debug("Starting rtp socket"); + auto rtp_task_config = espp::Task::Config{ + .name = "Rtp", + .callback = nullptr, + .stack_size_bytes = 12 * 1024, + }; + auto rtp_config = espp::UdpSocket::ReceiveConfig{ + .port = rtp_port, + .buffer_size = 2048, + .on_receive_callback = std::bind(&RtspClient::handle_rtp_packet, this, std::placeholders::_1, std::placeholders::_2), + }; + if (!rtp_socket_.start_receiving(rtp_task_config, rtp_config)) { + ec = std::make_error_code(std::errc::operation_canceled); + logger_.error("Failed to start receiving rtp packets"); + return; + } + } + + /// Initialize the RTCP socket + /// \note Starts the RTCP socket task. + /// \param rtcp_port The RTCP client port + /// \param ec The error code to set if an error occurs + void init_rtcp(size_t rtcp_port, std::error_code& ec) { + // exit early if the error code is set + if (ec) { + return; + } + logger_.debug("Starting rtcp socket"); + auto rtcp_task_config = espp::Task::Config{ + .name = "Rtcp", + .callback = nullptr, + .stack_size_bytes = 12 * 1024, + }; + auto rtcp_config = espp::UdpSocket::ReceiveConfig{ + .port = rtcp_port, + .buffer_size = 2048, + .on_receive_callback = std::bind(&RtspClient::handle_rtcp_packet, this, std::placeholders::_1, std::placeholders::_2), + }; + if (!rtcp_socket_.start_receiving(rtcp_task_config, rtcp_config)) { + ec = std::make_error_code(std::errc::operation_canceled); + logger_.error("Failed to start receiving rtcp packets"); + return; + } + } + + /// Handle an RTP packet + /// \note Parses the RTP packet and appends it to the current JPEG frame. + /// \note If the packet is the last fragment of the JPEG frame, the frame is sent to the on_jpeg_frame callback. + /// \note This function is called by the RTP socket task. + /// \param data The data to handle + /// \param sender_info The sender info + /// \return Optional data to send back to the sender + std::optional> handle_rtp_packet(std::vector &data, const espp::Socket::Info &sender_info) { + // jpeg frame that we are building + static std::unique_ptr jpeg_frame; + + std::string_view packet(reinterpret_cast(data.data()), data.size()); + // parse the rtp packet + RtpJpegPacket rtp_jpeg_packet(packet); + auto frag_offset = rtp_jpeg_packet.get_offset(); + if (frag_offset == 0) { + // first fragment + jpeg_frame = std::make_unique(rtp_jpeg_packet); + } else if (jpeg_frame) { + // middle fragment + jpeg_frame->append(rtp_jpeg_packet); + } else { + // we don't have a frame to append to but we got a middle fragment + // this is an error + logger_.warn("Received middle fragment without a frame"); + return {}; + } + + // check if this is the last packet of the frame (the last packet will have + // the marker bit set) + if (jpeg_frame && rtp_jpeg_packet.get_marker()) { + // serialize the jpeg frame + jpeg_frame->serialize(); + // get the jpeg data + auto jpeg_data = jpeg_frame->get_data(); + logger_.debug("Received jpeg frame of size: {}", jpeg_data.size()); + // call the on_jpeg_frame callback + if (on_jpeg_frame_) { + on_jpeg_frame_(std::move(jpeg_frame)); + } + } + // return an empty vector to indicate that we don't want to send a response + return {}; + } + + /// Handle an RTCP packet + /// \note Parses the RTCP packet and sends a response if necessary. + /// \note This function is called by the RTCP socket task. + /// \param data The data to handle + /// \param sender_info The sender info + /// \return Optional data to send back to the sender + std::optional> handle_rtcp_packet(std::vector &data, const espp::Socket::Info &sender_info) { + // receive the rtcp packet + std::string_view packet(reinterpret_cast(data.data()), data.size()); + // TODO: parse the rtcp packet + // return an empty vector to indicate that we don't want to send a response + return {}; + } + + std::string server_address_; + int rtsp_port_; + + espp::TcpSocket rtsp_socket_; + espp::UdpSocket rtp_socket_; + espp::UdpSocket rtcp_socket_; + + jpeg_frame_callback_t on_jpeg_frame_{nullptr}; + + int cseq_ = 0; + int video_port_ = 0; + int video_payload_type_ = 0; + std::string path_; + std::string session_id_; + + espp::Logger logger_; + }; + +} // namespace espp diff --git a/components/rtsp_client/python/rtsp_client.py b/components/rtsp_client/python/rtsp_client.py index 8b80569..7b5b83e 100644 --- a/components/rtsp_client/python/rtsp_client.py +++ b/components/rtsp_client/python/rtsp_client.py @@ -317,7 +317,10 @@ def handle_rtp_packet(self): # Process RTP packet in rtp_data rtp_data, addr = self.rtp_socket.recvfrom(8192) rtp_packet = RtpJpegPacket(rtp_data) - jpeg_data = rtp_packet.get_jpeg_data() + + # TODO: handle out of order packets + # (packets whose seq_num is not equal to the previous packet's + # seq_num) frag_offset = rtp_packet.get_frag_offset() if frag_offset == 0: From 17e5b786fd040b9e4d07612add7add90e14cf20a Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Sat, 22 Apr 2023 16:07:31 -0500 Subject: [PATCH 04/11] feat(rtsp_client): updated component * Changed jpeg_header to be object instead of unique ptr in jpeg frame * Added getters for width/height of image in jpeg header and jpeg frame * Updated parsing of mjpeg header rtp_jpeg_packet * Added receive timeout in rtsp_client - defaults to 5 seconds for now --- components/rtsp_client/include/jpeg_frame.hpp | 21 ++++++++++++++----- .../rtsp_client/include/jpeg_header.hpp | 12 +++++++++++ .../rtsp_client/include/rtp_jpeg_packet.hpp | 12 ++++++----- .../rtsp_client/include/rtsp_client.hpp | 1 + 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/components/rtsp_client/include/jpeg_frame.hpp b/components/rtsp_client/include/jpeg_frame.hpp index 740969b..899950c 100644 --- a/components/rtsp_client/include/jpeg_frame.hpp +++ b/components/rtsp_client/include/jpeg_frame.hpp @@ -17,13 +17,24 @@ namespace espp { /// data to the frame. /// /// @param packet The packet to parse. - explicit JpegFrame(const RtpJpegPacket& packet) { - // parse the header - header_ = std::make_unique(packet.get_width(), packet.get_height(), packet.get_q_table(0), packet.get_q_table(1)); + explicit JpegFrame(const RtpJpegPacket& packet) + : header_(packet.get_width(), packet.get_height(), packet.get_q_table(0), packet.get_q_table(1)) { // add the jpeg data scans_.push_back(packet.get_jpeg_data()); } + /// Get the width of the frame. + /// @return The width of the frame. + int get_width() const { + return header_.get_width(); + } + + /// Get the height of the frame. + /// @return The height of the frame. + int get_height() const { + return header_.get_height(); + } + /// Append a RtpJpegPacket to the frame. /// This will add the JPEG data to the frame. /// @param packet The packet containing the scan to append. @@ -54,7 +65,7 @@ namespace espp { /// @note This method must be called before get_data() can be used. /// @note This method should only be called once. void serialize() { - auto header_data = header_->get_data(); + auto header_data = header_.get_data(); auto scan_bytes = 0; for (auto& scan : scans_) { scan_bytes += scan.size(); @@ -86,7 +97,7 @@ namespace espp { } protected: - std::unique_ptr header_; + JpegHeader header_; std::vector scans_; std::vector data_; }; diff --git a/components/rtsp_client/include/jpeg_header.hpp b/components/rtsp_client/include/jpeg_header.hpp index f468fac..15c03c9 100644 --- a/components/rtsp_client/include/jpeg_header.hpp +++ b/components/rtsp_client/include/jpeg_header.hpp @@ -23,6 +23,18 @@ namespace espp { ~JpegHeader() {} + /// Get the image width. + /// @return The image width in pixels. + int get_width() const { + return width_; + } + + /// Get the image height. + /// @return The image height in pixels. + int get_height() const { + return height_; + } + /// Get the JPEG header data. /// @return The JPEG header data. std::string_view get_data() { diff --git a/components/rtsp_client/include/rtp_jpeg_packet.hpp b/components/rtsp_client/include/rtp_jpeg_packet.hpp index 5f37f52..456f268 100644 --- a/components/rtsp_client/include/rtp_jpeg_packet.hpp +++ b/components/rtsp_client/include/rtp_jpeg_packet.hpp @@ -44,12 +44,13 @@ namespace espp { static constexpr int Q_TABLE_SIZE = 64; void parse_mjpeg_header() { - int offset = RTP_HEADER_SIZE + MJPEG_HEADER_SIZE; + int offset = RTP_HEADER_SIZE; type_specific_ = packet_[offset]; - offset_ = (packet_[offset + 1] << 8) | packet_[offset + 2]; - q_ = packet_[offset + 3]; - width_ = (packet_[offset + 4] << 8) | packet_[offset + 5]; - height_ = (packet_[offset + 6] << 8) | packet_[offset + 7]; + offset_ = (packet_[offset + 1] << 16) | (packet_[offset + 2] << 8) | packet_[offset + 3]; + frag_type_ = packet_[offset + 4]; + q_ = packet_[offset + 5]; + width_ = packet_[offset + 6] * 8; + height_ = packet_[offset + 7] * 8; // If the Q value is between 128 and 256, then the packet contains // quantization tables. @@ -67,6 +68,7 @@ namespace espp { int type_specific_; int offset_; + int frag_type_; int q_; int width_; int height_; diff --git a/components/rtsp_client/include/rtsp_client.hpp b/components/rtsp_client/include/rtsp_client.hpp index a1d5eaa..11b292a 100644 --- a/components/rtsp_client/include/rtsp_client.hpp +++ b/components/rtsp_client/include/rtsp_client.hpp @@ -90,6 +90,7 @@ namespace espp { .wait_for_response = true, .response_size = 1024, .on_response_callback = [&response](auto &response_vector) { response.assign(response_vector.begin(), response_vector.end()); }, + .response_timeout = std::chrono::seconds(5), }; // NOTE: now this call blocks until the response is received logger_.debug("Request:\n{}", request); From 107ca713b04837983458f2ade58d83365bb98b92 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Sat, 22 Apr 2023 16:09:17 -0500 Subject: [PATCH 05/11] feat(main): update to use rtsp client * Removed older camera display code and tasks, updated to use new rtsp_client component * removed multicast socket for now for hardcoded server ip / port for testing. * Added rtsp_client to cmakelists for main --- CMakeLists.txt | 2 +- main/main.cpp | 249 +++++++++++++------------------------------------ 2 files changed, 66 insertions(+), 185 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eb6a5bc..b7bc8c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ add_compile_definitions(BOARD_HAS_PSRAM) set( COMPONENTS - "main esptool_py esp_psram jpegdec task format monitor display_drivers wifi socket" + "main esptool_py esp_psram jpegdec task format monitor display_drivers wifi socket rtsp_client" CACHE STRING "List of components to include" ) diff --git a/main/main.cpp b/main/main.cpp index 8c5f35c..ecb4cad 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -2,6 +2,7 @@ #include #include +#include #include "freertos/FreeRTOS.h" #include "freertos/task.h" @@ -17,6 +18,7 @@ #include "jpegdec.h" #include "format.hpp" +#include "rtsp_client.hpp" #include "task.hpp" #include "task_monitor.hpp" #include "tcp_socket.hpp" @@ -47,103 +49,13 @@ int drawMCUs(JPEGDRAW *pDraw) { return 1; } -struct Image { - uint8_t *data{nullptr}; - uint32_t num_bytes{0}; - size_t offset{0}; - int bytes_remaining{0}; -}; - -size_t get_image_length(const uint8_t* header) { - return - header[4] << 24 | - header[5] << 16 | - header[6] << 8 | - header[7]; -} - -void init_image(const uint8_t* data, size_t data_len, Image* image) { - // 4 start bytes, 4 bytes of image length in header before start of image data - static size_t data_offset = 8; - size_t jpeg_len = get_image_length(data); - size_t img_bytes_received = std::min(data_len - data_offset, jpeg_len); - image->num_bytes = jpeg_len; - image->data = (uint8_t*)heap_caps_malloc(image->num_bytes, MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); - // add the bytes to our new array - memcpy(image->data, &data[data_offset], img_bytes_received); - image->offset = img_bytes_received; - image->bytes_remaining = jpeg_len - img_bytes_received; -} - -void update_image(const uint8_t* data, size_t data_len, Image* image) { - memcpy(&image->data[image->offset], data, data_len); - image->bytes_remaining -= data_len; - image->offset += data_len; - // fmt::print("Updated image offset = {}, remaining = {}/{}\n", - // image->offset, image->bytes_remaining, image->num_bytes); -} - -int find_header(std::basic_string_view data) { - static std::vector header{0xAA, 0xBB, 0xCC, 0xDD}; - auto res = std::search(std::begin(data), std::end(data), std::begin(header), std::end(header)); - bool has_header = res != std::end(data); - if (!has_header) { - return -1; - } - size_t header_offset = res - std::begin(data); - return header_offset; -} - -int handle_image_data(std::basic_string_view data, QueueHandle_t image_queue) { - static bool has_seen_header = false; - static Image image0, image1; - static size_t image_index = 0; - Image* image = image_index ? &image1 : &image0; - // will need to put multiple packets together into a single image based on - // header + length. only copy / allocate if there is space in the queue, - // otherwise just discard this image. - if (!has_seen_header) { - auto header_offset = find_header(data); - if (header_offset < 0) return -1; - size_t length = data.size() - header_offset; - init_image(&data[header_offset], length, image); - // fmt::print("Got header at offset {}/{}, remaining = {}/{}\n", - // header_offset, data.size(), image->bytes_remaining, image->num_bytes); - // update state - has_seen_header = true; - } else { - // we've seen the header, so the beginning of this packet must be the - // continuation of the image. - size_t num_bytes = std::min(image->bytes_remaining, (int)data.size()); - // fmt::print("continuation: offset = {}, data length = {}/{}, remaining = {}/{}\n", - // image->offset, num_bytes, data.size(), image->bytes_remaining, image->num_bytes); - update_image(data.data(), num_bytes, image); - } - - // return bytes remaining - if (image->bytes_remaining > 0) { - return image->bytes_remaining; - } - - // bytes_remaining is <= 0, so send the image - auto num_spots = uxQueueSpacesAvailable(image_queue); - if (num_spots > 0) { - xQueueSend(image_queue, image, portMAX_DELAY); - image_index = image_index ? 0 : 1; - image = image_index ? &image1 : &image0; - } else { - free(image->data); - image->data = nullptr; - } - has_seen_header = false; - return -1; -} - extern "C" void app_main(void) { - esp_err_t err; espp::Logger logger({.tag = "Camera Display", .level = espp::Logger::Verbosity::INFO}); logger.info("Bootup"); + // initialize the lcd for the image display + lcd_init(); + // initialize NVS, needed for WiFi esp_err_t ret = nvs_flash_init(); if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) { @@ -172,66 +84,47 @@ extern "C" void app_main(void) { std::this_thread::sleep_for(1s); } - // multicast our receiver info over UDP - // create threads - auto client_task_fn = [](auto&, auto&) { - static espp::UdpSocket client_socket({}); - static std::string multicast_group = "239.1.1.1"; - static size_t multicast_port = 5000; - static std::string payload = "hello world"; - static auto send_config = espp::UdpSocket::SendConfig{ - .ip_address = multicast_group, - .port = multicast_port, - .is_multicast_endpoint = true, - }; - // NOTE: now this call blocks until the response is received - client_socket.send(payload, send_config); - std::this_thread::sleep_for(1s); - }; - auto client_task = espp::Task::make_unique({ - .name = "Client Task", - .callback = client_task_fn, - .stack_size_bytes = 3*1024 - }); - client_task->start(); - - // initialize the lcd for the image display - lcd_init(); + std::mutex jpeg_mutex; + std::condition_variable jpeg_cv; + static constexpr size_t MAX_JPEG_FRAMES = 2; + std::deque> jpeg_frames; // create the parsing and display task logger.info("Creating display task"); std::atomic num_frames_displayed{0}; - QueueHandle_t receive_queue = xQueueCreate(2, sizeof(Image)); std::atomic elapsed{0}; - auto display_task_fn = [&receive_queue, &num_frames_displayed, &elapsed](auto& m, auto& cv) { + auto display_task_fn = [&jpeg_mutex, &jpeg_cv, &jpeg_frames, &num_frames_displayed, &elapsed](auto& m, auto& cv) { // the original (max) image size is 1600x1200, but the S3 BOX has a resolution of 320x240 // wait on the queue until we have an image ready to display - static Image image; static JPEGDEC jpeg; static espp::Logger logger({.tag = "Decoder", .level = espp::Logger::Verbosity::INFO}); - if (xQueueReceive(receive_queue, &image, portMAX_DELAY) == pdPASS) { - logger.debug("Got image, length = {}", image.num_bytes); - static auto start = std::chrono::high_resolution_clock::now(); - if (jpeg.openRAM(image.data, image.num_bytes, drawMCUs)) { - logger.debug("Image size: {} x {}, orientation: {}, bpp: {}", jpeg.getWidth(), - jpeg.getHeight(), jpeg.getOrientation(), jpeg.getBpp()); - jpeg.setPixelType(RGB565_BIG_ENDIAN); - if (!jpeg.decode(0,0,0)) { - logger.error("Error decoding"); - } - } else { - logger.error("error opening jpeg image"); + std::unique_ptr image; + { + // wait for a frame to be available + std::unique_lock lock(jpeg_mutex); + jpeg_cv.wait(lock, [&jpeg_frames] { return !jpeg_frames.empty(); }); + image = std::move(jpeg_frames.front()); + jpeg_frames.pop_front(); + } + static auto start = std::chrono::high_resolution_clock::now(); + auto image_data = image->get_data(); + logger.info("Decoding image of size {}", image_data.size()); + logger.info("Decoding image, shape = {} x {}", image->get_width(), image->get_height()); + if (jpeg.openRAM((uint8_t*)(image_data.data()), image_data.size(), drawMCUs)) { + logger.debug("Image size: {} x {}, orientation: {}, bpp: {}", jpeg.getWidth(), + jpeg.getHeight(), jpeg.getOrientation(), jpeg.getBpp()); + jpeg.setPixelType(RGB565_BIG_ENDIAN); + if (!jpeg.decode(0,0,0)) { + logger.error("Error decoding"); } - - auto end = std::chrono::high_resolution_clock::now(); - elapsed = std::chrono::duration(end-start).count(); - num_frames_displayed += 1; - - // now free the memory we allocated when receiving the jpeg buffer - free(image.data); + } else { + logger.error("error opening jpeg image"); } + auto end = std::chrono::high_resolution_clock::now(); + elapsed = std::chrono::duration(end-start).count(); + num_frames_displayed += 1; }; // Start the display task logger.info("Starting display task"); @@ -245,52 +138,40 @@ extern "C" void app_main(void) { // make the tcp_server logger.info("Starting server task"); std::atomic num_frames_received{0}; - espp::TcpSocket server_socket({.log_level=espp::Logger::Verbosity::WARN}); - static constexpr size_t port = 8888; - // bind - if (!server_socket.bind(port)){ - return; - } - // listen - static constexpr size_t max_pending_connections = 1; - if (!server_socket.listen(max_pending_connections)) { - return; - } - auto server_task = espp::Task::make_unique({ - .name = "TcpServer Task", - .callback = [&server_socket, &receive_queue, &num_frames_received](auto& m, auto& cv) { - static espp::Logger logger({.tag = "Receiver", .level = espp::Logger::Verbosity::INFO}); - // ensure our socket is already closed - server_socket.close_accepted_socket(); - // accept - if (!server_socket.accept()) { - return; - } - // receive - auto client_socket = server_socket.get_accepted_socket(); - static constexpr size_t max_buffer_size = 1024; - static uint8_t *data = (uint8_t*)heap_caps_malloc(max_buffer_size, MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); - size_t receive_buffer_size = max_buffer_size; - while (true) { - memset(data, 0, max_buffer_size); - logger.debug("Trying to receive {} B", receive_buffer_size); - auto num_bytes = server_socket.receive(client_socket, receive_buffer_size, data); - if (num_bytes < 0) { - // couldn't receive, let's see if we can break to try to accept again - break; - } - int bytes_remaining = handle_image_data(std::basic_string_view(data, num_bytes), receive_queue); - if (bytes_remaining > 0) { - receive_buffer_size = std::min(bytes_remaining, (int)max_buffer_size); - } else { - num_frames_received += 1; - receive_buffer_size = max_buffer_size; + espp::RtspClient rtsp_client({ + .server_address = "192.168.86.181", + .rtsp_port = 8554, + .path = "/mjpeg/1", + .on_jpeg_frame = [&jpeg_mutex, &jpeg_cv, &jpeg_frames, &num_frames_received](std::unique_ptr jpeg_frame) { + { + std::lock_guard lock(jpeg_mutex); + jpeg_frames.push_back(std::move(jpeg_frame)); } + jpeg_cv.notify_all(); + num_frames_received += 1; } - }, - .stack_size_bytes = 10 * 1024, - }); - server_task->start(); + }); + + std::error_code ec; + rtsp_client.connect(ec); + if (ec) { + logger.error("Error connecting to server: {}", ec.message()); + } + + rtsp_client.describe(ec); + if (ec) { + logger.error("Error describing server: {}", ec.message()); + } + + rtsp_client.setup(ec); + if (ec) { + logger.error("Error setting up server: {}", ec.message()); + } + + rtsp_client.play(ec); + if (ec) { + logger.error("Error playing server: {}", ec.message()); + } auto start = std::chrono::high_resolution_clock::now(); while (true) { From c9a6998b2143c04541c67ff118e06ec5d86729db Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Sun, 23 Apr 2023 11:23:51 -0500 Subject: [PATCH 06/11] feat(rtsp_client): fixed bugs and fleshed out * Updated how memory is managed in the jpeg frame to ensure consistency * Added missing huffman tables to header generation (copied from generated jpeg frames on the camera side) * Updated ordering of header generation to match frames generated * Updated how the frame completeness is tracked to make the code simpler to use --- components/rtsp_client/include/jpeg_frame.hpp | 89 +++++++++++-------- .../rtsp_client/include/jpeg_header.hpp | 60 ++++++------- .../rtsp_client/include/rtp_jpeg_packet.hpp | 49 +++++----- components/rtsp_client/include/rtp_packet.hpp | 6 +- .../rtsp_client/include/rtsp_client.hpp | 21 +++-- 5 files changed, 125 insertions(+), 100 deletions(-) diff --git a/components/rtsp_client/include/jpeg_frame.hpp b/components/rtsp_client/include/jpeg_frame.hpp index 899950c..9c71467 100644 --- a/components/rtsp_client/include/jpeg_frame.hpp +++ b/components/rtsp_client/include/jpeg_frame.hpp @@ -19,8 +19,10 @@ namespace espp { /// @param packet The packet to parse. explicit JpegFrame(const RtpJpegPacket& packet) : header_(packet.get_width(), packet.get_height(), packet.get_q_table(0), packet.get_q_table(1)) { + // add the jpeg header + serialize_header(); // add the jpeg data - scans_.push_back(packet.get_jpeg_data()); + add_scan(packet); } /// Get the width of the frame. @@ -35,6 +37,12 @@ namespace espp { return header_.get_height(); } + /// Check if the frame is complete. + /// @return True if the frame is complete, false otherwise. + bool is_complete() const { + return finalized_; + } + /// Append a RtpJpegPacket to the frame. /// This will add the JPEG data to the frame. /// @param packet The packet containing the scan to append. @@ -44,61 +52,64 @@ namespace espp { /// Append a JPEG scan to the frame. /// This will add the JPEG data to the frame. + /// @note If the packet contains the EOI marker, the frame will be + /// finalized, and no further scans can be added. /// @param packet The packet containing the scan to append. void add_scan(const RtpJpegPacket& packet) { add_scan(packet.get_jpeg_data()); + if (packet.get_marker()) { + finalize(); + } + } + + /// Get the serialized data. + /// This will return the serialized data. + /// @return The serialized data. + std::string_view get_data() const { + return std::string_view(data_.data(), data_.size()); + } + + protected: + /// Serialize the header. + void serialize_header() { + auto header_data = header_.get_data(); + data_.resize(header_data.size()); + memcpy(data_.data(), header_data.data(), header_data.size()); } /// Append a JPEG scan to the frame. /// This will add the JPEG data to the frame. /// @param scan The jpeg scan to append. void add_scan(std::string_view scan) { - scans_.push_back(scan); - } - - /// Serialize the frame. - /// - /// This will serialize the header and all scans into a single buffer which - /// can be sent over the network. You can get the serialized data using - /// get_data(). - /// - /// @note This method must be called before get_data() can be used. - /// @note This method should only be called once. - void serialize() { - auto header_data = header_.get_data(); - auto scan_bytes = 0; - for (auto& scan : scans_) { - scan_bytes += scan.size(); + if (finalized_) { + // TODO: handle this error + return; } - data_.resize(header_data.size() + scan_bytes + 2);; + data_.insert(std::end(data_), std::begin(scan), std::end(scan)); + } - int offset = 0; - // serialize the header - memcpy(data_.data(), header_data.data(), header_data.size()); - offset += header_data.size(); - // serialize the scans - for (auto& scan : scans_) { - memcpy(data_.data() + offset, scan.data(), scan.size()); - offset += scan.size(); + /// Add the EOI marker to the frame. + /// This will add the EOI marker to the frame. This must be called before + /// calling get_data(). + /// @note This will prevent any further scans from being added to the frame. + void finalize() { + if (!finalized_) { + finalized_ = true; + // add_eoi(); + } else { + // TODO: handle this error + // already finalized } - // add the EOI marker - data_[offset++] = 0xFF; - data_[offset++] = 0xD9; } - /// Get the serialized data. - /// - /// This will return the serialized data. You must call serialize() before - /// calling this method. - /// - /// @return The serialized data. - std::string_view get_data() const { - return std::string_view(data_.data(), data_.size()); + /// Add the EOI marker to the frame. + void add_eoi() { + data_.push_back(0xFF); + data_.push_back(0xD9); } - protected: JpegHeader header_; - std::vector scans_; + bool finalized_ = false; std::vector data_; }; } // namespace espp diff --git a/components/rtsp_client/include/jpeg_header.hpp b/components/rtsp_client/include/jpeg_header.hpp index 15c03c9..ac425e5 100644 --- a/components/rtsp_client/include/jpeg_header.hpp +++ b/components/rtsp_client/include/jpeg_header.hpp @@ -44,44 +44,37 @@ namespace espp { protected: static constexpr int SOF0_SIZE = 19; - static constexpr int DQT_HEADER_SIZE = 4; + static constexpr int DQT_HEADER_SIZE = 5; // JFIF APP0 Marker for version 1.2 with 72 DPI and no thumbnail static constexpr char JFIF_APP0_DATA[] = { 0xFF, 0xE0, // APP0 marker 0x00, 0x10, // Length of APP0 data (16 bytes) 0x4A, 0x46, 0x49, 0x46, 0x00, // Identifier: ASCII "JFIF\0" - 0x01, 0x02, // Version number (1.2) + 0x01, 0x01, // Version number (1.1) 0x01, // Units: 1 = dots per inch - 0x00, 0x48, // X density (72 DPI) - 0x00, 0x48, // Y density (72 DPI) + 0x00, 0x00, // X density + 0x00, 0x00, // Y density 0x00, 0x00 // No thumbnail }; static constexpr char HUFFMAN_TABLES[] = { - // 1st table - // Default luminance DC Huffman table - 0xff, 0xc4, 0x00, 0x1f, 0x00, // header - 0x00, 0x01, 0x05, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0a, 0x0b, - // 2nd table - // Default luminance AC Huffman table - 0xff, 0xc4, 0x00, 0xb5, 0x10, // header - 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01, 0x7d, 0x01, 0x02, - 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, - 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, - 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, - 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, - 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, - 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2, 0xe3, 0xe4, - 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, + // Huffman table DC (luminance) + 0xff, 0xc4, + 0x00, 0x1f, 0x00, + 0x00, 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + // Huffman table AC (luminance) + 0xff, 0xc4, + 0x00, 0xb5, 0x10, + 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01, 0x7d, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, + // Huffman table DC (chrominance) + 0xff, 0xc4, + 0x00, 0x1f, 0x01, + 0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + // Huffman table AC (chrominance) + 0xff, 0xc4, + 0x00, 0xb5, 0x11, + 0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04, 0x07, 0x05, 0x04, 0x04, 0x00, 0x01, 0x02, 0x77, 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, 0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0, 0x15, 0x62, 0x72, 0xd1, 0x0a, 0x16, 0x24, 0x34, 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, }; // Scan header (SOS) @@ -89,7 +82,10 @@ namespace espp { 0xFF, 0xDA, // SOS marker 0x00, 0x0C, // length 0x03, // number of components - 0x01, 0x00, 0x02, 0x11, 0x03, 0x11, 0x00, 0x3F, 0x00 // components + 0x01, 0x00, // component IDs and Huffman tables + 0x02, 0x11, // component IDs and Huffman tables + 0x03, 0x11, // component IDs and Huffman tables + 0x00, 0x3F, 0x00 // Ss, Se, Ah/Al }; @@ -133,8 +129,8 @@ namespace espp { q0_table_.size() + DQT_HEADER_SIZE + q1_table_.size() + - SOF0_SIZE + sizeof(HUFFMAN_TABLES) + + SOF0_SIZE + sizeof(SOS); // serialize the jpeg header to the data_ vector data_.resize(header_size); @@ -166,13 +162,13 @@ namespace espp { memcpy(data_.data() + offset, q1_table_.data(), q1_table_.size()); offset += q1_table_.size(); - // add the SOF0 - offset = add_sof0(offset); - // add huffman tables memcpy(data_.data() + offset, HUFFMAN_TABLES, sizeof(HUFFMAN_TABLES)); offset += sizeof(HUFFMAN_TABLES); + // add the SOF0 + offset = add_sof0(offset); + // add the SOS marker memcpy(data_.data() + offset, SOS, sizeof(SOS)); offset += sizeof(SOS); diff --git a/components/rtsp_client/include/rtp_jpeg_packet.hpp b/components/rtsp_client/include/rtp_jpeg_packet.hpp index 456f268..3b042de 100644 --- a/components/rtsp_client/include/rtp_jpeg_packet.hpp +++ b/components/rtsp_client/include/rtp_jpeg_packet.hpp @@ -19,59 +19,68 @@ namespace espp { int get_mjpeg_header_size() const { return MJPEG_HEADER_SIZE; } std::string_view get_mjpeg_header() { - return std::string_view(packet_.data() + RTP_HEADER_SIZE, MJPEG_HEADER_SIZE); + return std::string_view(get_payload().data(), MJPEG_HEADER_SIZE); } int get_q_table_size() const { return Q_TABLE_SIZE; } - int get_num_q_tables() const { return q_table_indices_.size(); } + int get_num_q_tables() const { return q_tables_.size(); } std::string_view get_q_table(int index) const { if (index < get_num_q_tables()) { - return std::string_view(packet_.data() + q_table_indices_[index], Q_TABLE_SIZE); + return q_tables_[index]; } return {}; } std::string_view get_jpeg_data() const { - int quant_size = get_num_q_tables() * Q_TABLE_SIZE; - return std::string_view(packet_.data() + RTP_HEADER_SIZE + MJPEG_HEADER_SIZE + quant_size, - packet_.size() - RTP_HEADER_SIZE - MJPEG_HEADER_SIZE - quant_size); + auto payload = get_payload(); + return std::string_view(payload.data() + jpeg_data_start_, jpeg_data_size_); } protected: static constexpr int MJPEG_HEADER_SIZE = 8; + static constexpr int QUANT_HEADER_SIZE = 4; static constexpr int NUM_Q_TABLES = 2; static constexpr int Q_TABLE_SIZE = 64; void parse_mjpeg_header() { - int offset = RTP_HEADER_SIZE; - type_specific_ = packet_[offset]; - offset_ = (packet_[offset + 1] << 16) | (packet_[offset + 2] << 8) | packet_[offset + 3]; - frag_type_ = packet_[offset + 4]; - q_ = packet_[offset + 5]; - width_ = packet_[offset + 6] * 8; - height_ = packet_[offset + 7] * 8; + auto payload = get_payload(); + type_specific_ = payload[0]; + offset_ = (payload[1] << 16) | (payload[2] << 8) | payload[3]; + frag_type_ = payload[4]; + q_ = payload[5]; + width_ = payload[6] * 8; + height_ = payload[7] * 8; + + size_t offset = MJPEG_HEADER_SIZE; // If the Q value is between 128 and 256, then the packet contains // quantization tables. if (128 <= q_ && q_ <= 256) { - int num_quant_bytes = packet_[offset + 11]; + int num_quant_bytes = payload[11]; int expected_num_quant_bytes = NUM_Q_TABLES * Q_TABLE_SIZE; if (num_quant_bytes == expected_num_quant_bytes) { - q_table_indices_.resize(NUM_Q_TABLES); + q_tables_.resize(NUM_Q_TABLES); + offset += QUANT_HEADER_SIZE; for (int i = 0; i < NUM_Q_TABLES; i++) { - q_table_indices_[i] = offset + 12 + (i * Q_TABLE_SIZE); + q_tables_[i] = std::string_view(payload.data() + offset, Q_TABLE_SIZE); + offset += Q_TABLE_SIZE; } } } + + jpeg_data_start_ = offset; + jpeg_data_size_ = payload.size() - jpeg_data_start_; } int type_specific_; int offset_; int frag_type_; - int q_; - int width_; - int height_; - std::vector q_table_indices_; + int q_{0}; + int width_{0}; + int height_{0}; + int jpeg_data_start_{0}; + int jpeg_data_size_{0}; + std::vector q_tables_; }; } // namespace espp diff --git a/components/rtsp_client/include/rtp_packet.hpp b/components/rtsp_client/include/rtp_packet.hpp index 707b5d5..851f163 100644 --- a/components/rtsp_client/include/rtp_packet.hpp +++ b/components/rtsp_client/include/rtp_packet.hpp @@ -31,19 +31,19 @@ namespace espp { /// Get a string_view of the whole packet. /// @return A string_view of the whole packet. - std::string_view get_packet() { + std::string_view get_data() const { return std::string_view(packet_.data(), packet_.size()); } /// Get a string_view of the RTP header. /// @return A string_view of the RTP header. - std::string_view get_packet_header() { + std::string_view get_packet_header() const { return std::string_view(packet_.data(), RTP_HEADER_SIZE); } /// Get a string_view of the payload. /// @return A string_view of the payload. - std::string_view get_payload() { + std::string_view get_payload() const { return std::string_view(packet_.data() + RTP_HEADER_SIZE, payload_size_); } diff --git a/components/rtsp_client/include/rtsp_client.hpp b/components/rtsp_client/include/rtsp_client.hpp index 11b292a..f5d9882 100644 --- a/components/rtsp_client/include/rtsp_client.hpp +++ b/components/rtsp_client/include/rtsp_client.hpp @@ -122,6 +122,7 @@ namespace espp { return; } + rtsp_socket_.reinit(); auto did_connect = rtsp_socket_.connect({ .ip_address = server_address_, .port = static_cast(rtsp_port_), @@ -338,7 +339,7 @@ namespace espp { }; auto rtp_config = espp::UdpSocket::ReceiveConfig{ .port = rtp_port, - .buffer_size = 2048, + .buffer_size = 6 * 1024, .on_receive_callback = std::bind(&RtspClient::handle_rtp_packet, this, std::placeholders::_1, std::placeholders::_2), }; if (!rtp_socket_.start_receiving(rtp_task_config, rtp_config)) { @@ -365,7 +366,7 @@ namespace espp { }; auto rtcp_config = espp::UdpSocket::ReceiveConfig{ .port = rtcp_port, - .buffer_size = 2048, + .buffer_size = 6 * 1024, .on_receive_callback = std::bind(&RtspClient::handle_rtcp_packet, this, std::placeholders::_1, std::placeholders::_2), }; if (!rtcp_socket_.start_receiving(rtcp_task_config, rtcp_config)) { @@ -392,8 +393,17 @@ namespace espp { auto frag_offset = rtp_jpeg_packet.get_offset(); if (frag_offset == 0) { // first fragment + logger_.debug("Received first fragment, size: {}, sequence number: {}", + rtp_jpeg_packet.get_data().size(), rtp_jpeg_packet.get_sequence_number()); + if (jpeg_frame) { + // we already have a frame, this is an error + logger_.warn("Received first fragment but already have a frame"); + jpeg_frame.reset(); + } jpeg_frame = std::make_unique(rtp_jpeg_packet); } else if (jpeg_frame) { + logger_.debug("Received middle fragment, size: {}, sequence number: {}", + rtp_jpeg_packet.get_data().size(), rtp_jpeg_packet.get_sequence_number()); // middle fragment jpeg_frame->append(rtp_jpeg_packet); } else { @@ -405,16 +415,15 @@ namespace espp { // check if this is the last packet of the frame (the last packet will have // the marker bit set) - if (jpeg_frame && rtp_jpeg_packet.get_marker()) { - // serialize the jpeg frame - jpeg_frame->serialize(); + if (jpeg_frame && jpeg_frame->is_complete()) { // get the jpeg data auto jpeg_data = jpeg_frame->get_data(); - logger_.debug("Received jpeg frame of size: {}", jpeg_data.size()); + logger_.debug("Received jpeg frame of size: {} B", jpeg_data.size()); // call the on_jpeg_frame callback if (on_jpeg_frame_) { on_jpeg_frame_(std::move(jpeg_frame)); } + logger_.debug("Sent jpeg frame to callback, now jpeg_frame is nullptr? {}", jpeg_frame == nullptr); } // return an empty vector to indicate that we don't want to send a response return {}; From dc33f0ef6ced073397b5d3f2303bc79676017dbe Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Sun, 23 Apr 2023 11:24:49 -0500 Subject: [PATCH 07/11] example(rtsp_client): updated example * Keep retrying connection if it fails * Print last frame after connection termination --- .../example/main/rtsp_client_example.cpp | 24 ++++++++++++++----- components/rtsp_client/example/partitions.csv | 10 ++++---- .../rtsp_client/example/sdkconfig.defaults | 6 +++++ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/components/rtsp_client/example/main/rtsp_client_example.cpp b/components/rtsp_client/example/main/rtsp_client_example.cpp index 15742d8..65e5c83 100644 --- a/components/rtsp_client/example/main/rtsp_client_example.cpp +++ b/components/rtsp_client/example/main/rtsp_client_example.cpp @@ -51,24 +51,31 @@ extern "C" void app_main(void) { std::this_thread::sleep_for(1s); } + std::unique_ptr most_recent_jpeg_frame; std::error_code ec; espp::RtspClient rtsp_client({ .server_address = "192.168.86.181", .rtsp_port = 8554, .path = "/mjpeg/1", - .on_jpeg_frame = [](std::unique_ptr jpeg_frame) { + .on_jpeg_frame = [&most_recent_jpeg_frame](std::unique_ptr jpeg_frame) { auto jpeg_data = jpeg_frame->get_data(); auto jpeg_size = jpeg_data.size(); fmt::print("Got JPEG frame: {} bytes\n", jpeg_size); + most_recent_jpeg_frame = std::move(jpeg_frame); }, .log_level = espp::Logger::Verbosity::INFO, }); - rtsp_client.connect(ec); - if (ec) { - logger.error("Failed to connect to RTSP server: {}", ec.message()); - return; - } + do { + // clear the error code + ec.clear(); + rtsp_client.connect(ec); + if (ec) { + logger.error("Error connecting to server: {}", ec.message()); + logger.info("Retrying in 1s..."); + std::this_thread::sleep_for(1s); + } + } while (ec); rtsp_client.describe(ec); if (ec) { @@ -99,6 +106,11 @@ extern "C" void app_main(void) { // ignore the error here rtsp_client.disconnect(ec); + auto jpeg_data = most_recent_jpeg_frame->get_data(); + logger.info("Most recent JPEG frame: {} bytes", jpeg_data.size()); + std::vector jpeg_vector(jpeg_data.begin(), jpeg_data.end()); + logger.info("Most recent JPEG:\n{::#04x}", jpeg_vector); + logger.info("RtspClient example finished"); // wait forever diff --git a/components/rtsp_client/example/partitions.csv b/components/rtsp_client/example/partitions.csv index c4217ab..a34abb5 100644 --- a/components/rtsp_client/example/partitions.csv +++ b/components/rtsp_client/example/partitions.csv @@ -1,5 +1,5 @@ -# ESP-IDF Partition Table -# Name, Type, SubType, Offset, Size, Flags -nvs, data, nvs, 0x9000, 0x6000, -phy_init, data, phy, 0xf000, 0x1000, -factory, app, factory, 0x10000, 1500K, +# Name, Type, SubType, Offset, Size +nvs, data, nvs, 0x9000, 0x6000 +phy_init, data, phy, 0xf000, 0x1000 +factory, app, factory, 0x10000, 4M +images, 0x40, 0x01, , 8M diff --git a/components/rtsp_client/example/sdkconfig.defaults b/components/rtsp_client/example/sdkconfig.defaults index 397db0b..fa9a15e 100644 --- a/components/rtsp_client/example/sdkconfig.defaults +++ b/components/rtsp_client/example/sdkconfig.defaults @@ -8,6 +8,12 @@ CONFIG_FREERTOS_HZ=1000 CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y CONFIG_ESPTOOLPY_FLASHSIZE="16MB" +# +# Partition Table +# +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" + # # Common ESP-related # From 375ae324e1d73e173b7b15c53ed36acdf8e560eb Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Sun, 23 Apr 2023 11:25:15 -0500 Subject: [PATCH 08/11] updated python code to match c++ code, added display frame code --- .../rtsp_client/python/display_frame.py | 27 +++++ components/rtsp_client/python/rtsp_client.py | 105 +++++++++++++----- 2 files changed, 103 insertions(+), 29 deletions(-) create mode 100644 components/rtsp_client/python/display_frame.py diff --git a/components/rtsp_client/python/display_frame.py b/components/rtsp_client/python/display_frame.py new file mode 100644 index 0000000..5770a23 --- /dev/null +++ b/components/rtsp_client/python/display_frame.py @@ -0,0 +1,27 @@ +import sys +import cv2 +import numpy as np + +jpeg_data = [0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xdb, 0x00, 0x43, 0x00, 0x0f, 0x0a, 0x0b, 0x0d, 0x0b, 0x09, 0x0f, 0x0d, 0x0c, 0x0d, 0x11, 0x10, 0x0f, 0x12, 0x17, 0x26, 0x18, 0x17, 0x15, 0x15, 0x17, 0x2e, 0x21, 0x23, 0x1b, 0x26, 0x36, 0x30, 0x39, 0x38, 0x35, 0x30, 0x35, 0x34, 0x3c, 0x44, 0x56, 0x49, 0x3c, 0x40, 0x52, 0x41, 0x34, 0x35, 0x4b, 0x66, 0x4c, 0x52, 0x59, 0x5c, 0x61, 0x62, 0x61, 0x3a, 0x48, 0x6a, 0x71, 0x69, 0x5e, 0x71, 0x56, 0x5f, 0x61, 0x5d, 0xff, 0xdb, 0x00, 0x43, 0x01, 0x10, 0x11, 0x11, 0x17, 0x14, 0x17, 0x2c, 0x18, 0x18, 0x2c, 0x5d, 0x3e, 0x35, 0x3e, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0xff, 0xc4, 0x00, 0x1f, 0x00, 0x00, 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xc4, 0x00, 0xb5, 0x10, 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01, 0x7d, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xff, 0xc4, 0x00, 0x1f, 0x01, 0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xc4, 0x00, 0xb5, 0x11, 0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04, 0x07, 0x05, 0x04, 0x04, 0x00, 0x01, 0x02, 0x77, 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, 0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0, 0x15, 0x62, 0x72, 0xd1, 0x0a, 0x16, 0x24, 0x34, 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xff, 0xc0, 0x00, 0x11, 0x08, 0x00, 0xf0, 0x01, 0x40, 0x03, 0x01, 0x21, 0x00, 0x02, 0x11, 0x01, 0x03, 0x11, 0x01, 0xff, 0xda, 0x00, 0x0c, 0x03, 0x01, 0x00, 0x02, 0x11, 0x03, 0x11, 0x00, 0x3f, 0x00, 0xe0, 0x29, 0x2a, 0xa6, 0x42, 0x13, 0xbd, 0x0d, 0xd6, 0xa0, 0xa1, 0x28, 0xa0, 0x05, 0xa7, 0x50, 0x03, 0x96, 0xa4, 0x15, 0x22, 0x1f, 0x45, 0x00, 0x2d, 0x21, 0xa9, 0x18, 0x95, 0x22, 0x54, 0xb0, 0x1c, 0x69, 0x86, 0xa4, 0x63, 0x69, 0xb5, 0x42, 0x12, 0x92, 0x98, 0x05, 0x25, 0x30, 0x0a, 0x29, 0x80, 0x51, 0x40, 0x05, 0x14, 0x80, 0x29, 0x29, 0x88, 0x28, 0xa0, 0x02, 0x92, 0x80, 0x0a, 0x28, 0x10, 0x52, 0x53, 0x00, 0xa4, 0xad, 0x2a, 0x09, 0x0d, 0x3d, 0x69, 0x2a, 0x4b, 0x0a, 0x28, 0x01, 0xd4, 0xea, 0x00, 0x78, 0xa7, 0x8a, 0x91, 0x0e, 0xa5, 0xa4, 0x21, 0x69, 0x29, 0x14, 0x25, 0x48, 0xbd, 0x2a, 0x58, 0x01, 0xa6, 0x54, 0x80, 0xd3, 0x49, 0x56, 0x31, 0x29, 0x29, 0x88, 0x4a, 0x29, 0x80, 0x51, 0x40, 0x05, 0x25, 0x00, 0x14, 0xb4, 0x08, 0x29, 0x28, 0x00, 0xa2, 0x80, 0x0a, 0x28, 0x01, 0x28, 0xa0, 0x41, 0x49, 0x4c, 0x00, 0xd3, 0x7b, 0x56, 0xb5, 0x37, 0x12, 0x1b, 0x45, 0x49, 0x61, 0x4b, 0x48, 0x07, 0x53, 0x85, 0x20, 0x1e, 0x29, 0xf5, 0x24, 0x8b, 0x4a, 0x28, 0x01, 0x68, 0xa4, 0x30, 0xa7, 0x76, 0xa8, 0x63, 0x10, 0xd3, 0x68, 0x18, 0xca, 0x4a, 0xa0, 0x12, 0x8a, 0x62, 0x12, 0x8a, 0x00, 0x28, 0xa6, 0x01, 0x45, 0x00, 0x14, 0x94, 0x08, 0x5a, 0x4a, 0x40, 0x14, 0x53, 0x00, 0xa2, 0x80, 0x12, 0x8a, 0x04, 0x14, 0x94, 0x00, 0xa6, 0x90, 0x0c, 0x83, 0x5a, 0xd4, 0xdc, 0x48, 0x8e, 0x8a, 0x45, 0x85, 0x3a, 0x90, 0x0a, 0x29, 0xf4, 0x84, 0x3c, 0x0a, 0x76, 0x2a, 0x44, 0x2d, 0x14, 0x0c, 0x5a, 0x5a, 0x43, 0x0a, 0x7d, 0x43, 0x01, 0xa6, 0x99, 0x40, 0xc4, 0xa6, 0xd5, 0x08, 0x29, 0x29, 0x80, 0x52, 0x50, 0x01, 0x45, 0x30, 0x0a, 0x28, 0x00, 0xa2, 0x90, 0x05, 0x14, 0x00, 0x94, 0x53, 0x10, 0x51, 0x40, 0x09, 0x45, 0x00, 0x14, 0x50, 0x21, 0x1a, 0x97, 0xfe, 0x59, 0x9a, 0xd2, 0x7b, 0x82, 0x22, 0xa2, 0x82, 0x85, 0xa7, 0x52, 0x01, 0xc2, 0x9d, 0x48, 0x43, 0xc5, 0x3c, 0x54, 0x88, 0x28, 0xa0, 0x61, 0x4b, 0xda, 0x90, 0xc2, 0x9d, 0x50, 0xc0, 0x69, 0xa6, 0xd0, 0x31, 0x29, 0x2a, 0x84, 0x25, 0x25, 0x30, 0x0a, 0x28, 0x01, 0x28, 0xa0, 0x02, 0x8a, 0x60, 0x14, 0x50, 0x01, 0x45, 0x02, 0x0a, 0x4a, 0x00, 0x28, 0xa0, 0x04, 0xa2, 0x80, 0x0a, 0x28, 0x10, 0x8d, 0x48, 0x7e, 0xed, 0x69, 0x2d, 0xc1, 0x0c, 0xa2, 0x82, 0x87, 0x52, 0xd2, 0x01, 0xc2, 0xa4, 0x15, 0x22, 0x1c, 0x29, 0xf4, 0x84, 0x2d, 0x26, 0x29, 0x0c, 0x29, 0x09, 0xa0, 0x61, 0x8c, 0x8a, 0x5e, 0x40, 0xe9, 0x52, 0xc6, 0x37, 0x75, 0x25, 0x00, 0x14, 0xda, 0x60, 0x14, 0x94, 0xc4, 0x14, 0x50, 0x01, 0x49, 0x40, 0x05, 0x14, 0xc0, 0x28, 0xa0, 0x02, 0x8a, 0x04, 0x14, 0x94, 0x00, 0x51, 0x40, 0x05, 0x25, 0x00, 0x14, 0x50, 0x21, 0x0d, 0x0d, 0xf7, 0x6a, 0xde, 0xe3, 0x23, 0xa5, 0x14, 0x0c, 0x75, 0x2d, 0x20, 0x1e, 0x29, 0xf4, 0x84, 0x3c, 0x53, 0x85, 0x21, 0x0b, 0x45, 0x20, 0x12, 0x90, 0xa7, 0x34, 0x8a, 0x1c, 0x31, 0x41, 0xa8, 0x18, 0xd3, 0x4c, 0x20, 0x55, 0x00, 0x98, 0xa6, 0x9c, 0xd3, 0x00, 0x1d, 0x28, 0xa6, 0x20, 0xa2, 0x80, 0x0a, 0x28, 0x01, 0x28, 0xa6, 0x01, 0x45, 0x00, 0x14, 0x50, 0x20, 0xa2, 0x90, 0x09, 0x45, 0x30, 0x0a, 0x4a, 0x00, 0x28, 0xa0, 0x42, 0x77, 0xa1, 0xba, 0x55, 0x0c, 0x8e, 0x94, 0x53, 0x18, 0xea, 0x75, 0x20, 0x1e, 0x29, 0xe2, 0x90, 0x87, 0x8a, 0x75, 0x21, 0x05, 0x2d, 0x21, 0x89, 0x45, 0x21, 0x8a, 0x07, 0x14, 0xd2, 0xa2, 0xa0, 0x63, 0x71, 0xef, 0x4d, 0xe6, 0xa8, 0x04, 0xfc, 0x29, 0xac, 0x69, 0x80, 0x0e, 0x94, 0xb4, 0xc0, 0x4a, 0x28, 0x10, 0x51, 0x40, 0x09, 0x45, 0x00, 0x14, 0x53, 0x00, 0xa2, 0x90, 0x82, 0x8a, 0x00, 0x29, 0x29, 0x80, 0x51, 0x40, 0x09, 0x45, 0x02, 0x01, 0xd6, 0xa4, 0x23, 0x72, 0xf6, 0xa6, 0xc0, 0xad, 0x4e, 0x15, 0x45, 0x0e, 0x14, 0xea, 0x42, 0x1e, 0x2a, 0x41, 0x48, 0x43, 0x85, 0x3a, 0xa4, 0x05, 0xa2, 0x81, 0x89, 0x49, 0x52, 0x31, 0xc2, 0x9a, 0x6a, 0x06, 0x36, 0x9b, 0x56, 0x02, 0x53, 0x4f, 0x34, 0xc0, 0x28, 0xa6, 0x02, 0x51, 0x40, 0x82, 0x8a, 0x00, 0x28, 0xa0, 0x04, 0xa2, 0x80, 0x0a, 0x28, 0x00, 0xa2, 0x81, 0x09, 0x45, 0x00, 0x14, 0x94, 0xc0, 0x28, 0xa0, 0x42, 0x53, 0x81, 0x38, 0xa6, 0xc6, 0x46, 0x7a, 0xd2, 0x8a, 0xa1, 0x8e, 0x14, 0xfa, 0x91, 0x0e, 0x15, 0x20, 0xa4, 0x21, 0xf4, 0xb4, 0x80, 0x28, 0xa0, 0x61, 0x49, 0x52, 0x31, 0x69, 0x0d, 0x40, 0xc6, 0x53, 0x6a, 0xc0, 0x4a, 0x4a, 0x60, 0x25, 0x14, 0x00, 0x52, 0x53, 0x10, 0x51, 0x40, 0x05, 0x14, 0x00, 0x94, 0xb4, 0xc0, 0x28, 0xa4, 0x02, 0x51, 0x40, 0x82, 0x92, 0x98, 0x05, 0x14, 0x00, 0x94, 0x50, 0x21, 0x70, 0x36, 0x50, 0xbd, 0x28, 0x28, 0x47, 0x14, 0xd1, 0x54, 0x80, 0x75, 0x3e, 0x90, 0x87, 0x0a, 0x90, 0x52, 0x10, 0xfa, 0x5a, 0x40, 0x2d, 0x25, 0x03, 0x16, 0x92, 0xa4, 0x64, 0xf1, 0xdb, 0x4b, 0x28, 0x6f, 0x2e, 0x36, 0x7d, 0xab, 0xb8, 0xe0, 0x74, 0x15, 0x03, 0x54, 0x0c, 0x8e, 0x9b, 0x54, 0x02, 0x52, 0x55, 0x00, 0x52, 0x50, 0x01, 0x45, 0x31, 0x05, 0x25, 0x00, 0x14, 0x50, 0x01, 0x45, 0x00, 0x14, 0x50, 0x21, 0x28, 0xa0, 0x02, 0x92, 0x80, 0x0a, 0x29, 0x80, 0x52, 0x50, 0x21, 0xfd, 0x56, 0x85, 0xe9, 0x40, 0xc6, 0xc9, 0xd2, 0x99, 0x54, 0x86, 0x2e, 0x69, 0xe1, 0xa8, 0x10, 0xf0, 0x69, 0xf5, 0x22, 0x1d, 0xcd, 0x1c, 0xd2, 0x18, 0xb4, 0xb4, 0x80, 0x5a, 0x3b, 0xd2, 0x19, 0xd0, 0x68, 0x5a, 0x95, 0xbd, 0x95, 0xb5, 0xe9, 0x9b, 0xef, 0x34, 0x78, 0x45, 0xfe, 0xf5, 0x60, 0x48, 0x69, 0x0c, 0x88, 0xd3, 0x69, 0x80, 0x94, 0x94, 0xc0, 0x29, 0x28, 0x00, 0xa2, 0x98, 0x84, 0xa2, 0x80, 0x0a, 0x28, 0x00, 0xa2, 0x80, 0x0a, 0x28, 0x10, 0x52, 0x50, 0x01, 0x49, 0x4c, 0x02, 0x8a, 0x00, 0x29, 0x28, 0x11, 0x20, 0xa2, 0x81, 0x8c, 0x91, 0x87, 0x4a, 0x8c, 0x55, 0x21, 0x8e, 0xa5, 0xa0, 0x43, 0xa9, 0xea, 0x4d, 0x21, 0x12, 0x06, 0xa5, 0xde, 0x2a, 0x40, 0x4d, 0xd4, 0xea, 0x45, 0x0b, 0x45, 0x20, 0x17, 0x34, 0xd2, 0x6a, 0x46, 0x32, 0x92, 0xa8, 0x04, 0xa4, 0xa6, 0x01, 0x49, 0x40, 0x05, 0x14, 0xc0, 0x29, 0x28, 0x10, 0x51, 0x40, 0x05, 0x14, 0x00, 0x51, 0x40, 0x82, 0x92, 0x80, 0x0a, 0x28, 0x01, 0x28, 0xa6, 0x01, 0x49, 0x40, 0x89, 0x29, 0x8e, 0xf8, 0xe9, 0x40, 0xc8, 0xa8, 0xab, 0x18, 0xb4, 0xb4, 0x80, 0x76, 0xea, 0x70, 0x7a, 0x42, 0x1d, 0x9a, 0x75, 0x20, 0x1d, 0x4e, 0xa9, 0x18, 0xb4, 0x52, 0x00, 0xa6, 0xd2, 0x18, 0x94, 0x94, 0xc0, 0x4a, 0x4a, 0x60, 0x14, 0x94, 0x00, 0x51, 0x4c, 0x41, 0x45, 0x00, 0x25, 0x14, 0x00, 0x51, 0x40, 0x05, 0x14, 0x08, 0x29, 0x28, 0x00, 0xa2, 0x80, 0x12, 0x8a, 0x60, 0x14, 0x50, 0x20, 0x76, 0xc5, 0x43, 0x54, 0x8a, 0x0a, 0x5a, 0x60, 0x14, 0xb9, 0xa0, 0x07, 0x66, 0x96, 0x90, 0x0b, 0xc5, 0x38, 0x52, 0x01, 0xf4, 0xea, 0x80, 0x12, 0x9e, 0x29, 0x0c, 0x5a, 0x6d, 0x20, 0x12, 0x92, 0x98, 0x09, 0x45, 0x30, 0x12, 0x8a, 0x00, 0x4a, 0x29, 0x88, 0x28, 0xa0, 0x02, 0x92, 0x80, 0x16, 0x8a, 0x00, 0x29, 0x28, 0x10, 0x51, 0x40, 0x09, 0x45, 0x00, 0x14, 0x94, 0x00, 0x51, 0x4c, 0x44, 0x44, 0xe6, 0x92, 0xac, 0xa1, 0x68, 0xa0, 0x02, 0x96, 0x80, 0x0a, 0x75, 0x20, 0x1e, 0x29, 0xd5, 0x20, 0x3a, 0x8c, 0xd2, 0x18, 0x66, 0x9c, 0xb5, 0x2c, 0x07, 0x52, 0x52, 0x01, 0x29, 0x29, 0x88, 0x4a, 0x4a, 0x60, 0x14, 0x50, 0x02, 0x51, 0x4c, 0x02, 0x8a, 0x00, 0x29, 0x28, 0x01, 0x68, 0xa0, 0x02, 0x92, 0x81, 0x05, 0x14, 0x00, 0x94, 0x50, 0x02, 0x51, 0x4c, 0x41, 0x45, 0x00, 0x43, 0x45, 0x59, 0x42, 0xd1, 0x40, 0x05, 0x2d, 0x00, 0x2d, 0x2d, 0x20, 0x17, 0x34, 0xb9, 0xa4, 0x02, 0xee, 0xa5, 0xcd, 0x21, 0x8e, 0xa7, 0x0a, 0x90, 0x1d, 0x45, 0x21, 0x09, 0x49, 0x4c, 0x04, 0xa2, 0x80, 0x12, 0x8a, 0x00, 0x29, 0x29, 0x80, 0x51, 0x40, 0x05, 0x14, 0x00, 0x51, 0x48, 0x41, 0x49, 0x4c, 0x02, 0x8a, 0x00, 0x4a, 0x28, 0x00, 0xa2, 0x81, 0x09, 0x45, 0x30, 0x21, 0xa5, 0xab, 0x28, 0x28, 0xa0, 0x02, 0x96, 0x80, 0x16, 0x8a, 0x40, 0x2d, 0x14, 0x86, 0x38, 0x53, 0xd5, 0x69, 0x01, 0x25, 0x15, 0x00, 0x3a, 0x92, 0x90, 0x82, 0x92, 0xa8, 0x04, 0xa4, 0xa0, 0x04, 0xa2, 0x98, 0x05, 0x1d, 0xe9, 0x00, 0x37, 0x5a, 0x4a, 0x60, 0x14, 0x50, 0x01, 0x45, 0x21, 0x05, 0x25, 0x30, 0x0a, 0x28, 0x01, 0x28, 0xa0, 0x02, 0x8a, 0x04, 0x25, 0x14, 0xc0, 0x86, 0x96, 0xac, 0xa0, 0xa2, 0x80, 0x0a, 0x5a, 0x00, 0x29, 0x69, 0x0c, 0x29, 0xe0, 0x52, 0x11, 0x28, 0x14, 0xea, 0x90, 0x16, 0x8a, 0x90, 0x16, 0x8a, 0x00, 0x4a, 0x4a, 0x04, 0x25, 0x25, 0x30, 0x12, 0x8a, 0x63, 0x0a, 0x4a, 0x00, 0x28, 0xa0, 0x02, 0x8a, 0x00, 0x28, 0xa4, 0x01, 0x49, 0x4c, 0x41, 0x45, 0x00, 0x25, 0x14, 0x00, 0x51, 0x40, 0x84, 0xa2, 0x98, 0x10, 0xd2, 0xd5, 0x94, 0x25, 0x2d, 0x00, 0x14, 0xb4, 0x80, 0x29, 0x68, 0x01, 0xc2, 0x9e, 0x2a, 0x40, 0x78, 0xa5, 0xa4, 0x03, 0xc5, 0x23, 0x54, 0x8c, 0x29, 0x68, 0x24, 0x4a, 0x28, 0x01, 0x29, 0xb4, 0xc0, 0x4a, 0x29, 0x8c, 0x28, 0xa0, 0x02, 0x8a, 0x04, 0x14, 0x52, 0x01, 0x28, 0xa6, 0x01, 0x49, 0x40, 0x05, 0x14, 0x00, 0x94, 0x50, 0x01, 0x49, 0x40, 0x82, 0x8a, 0x60, 0x43, 0x45, 0x59, 0x61, 0x4b, 0x40, 0x82, 0x96, 0x90, 0x05, 0x2d, 0x00, 0x2d, 0x38, 0x52, 0x19, 0x20, 0xa7, 0x0a, 0x90, 0x1d, 0x48, 0xdd, 0x6a, 0x44, 0x28, 0xa5, 0xa0, 0x41, 0x49, 0x40, 0x09, 0x49, 0x40, 0x09, 0x45, 0x30, 0x12, 0x96, 0x80, 0x12, 0x8a, 0x00, 0x28, 0xa0, 0x04, 0xa2, 0x80, 0x0a, 0x4a, 0x60, 0x14, 0x50, 0x02, 0x51, 0x40, 0x05, 0x25, 0x02, 0x0a, 0x29, 0x81, 0x0d, 0x15, 0x65, 0x0b, 0x45, 0x00, 0x14, 0xb4, 0x80, 0x29, 0x68, 0x18, 0xb4, 0xe1, 0x48, 0x09, 0x05, 0x3a, 0xa0, 0x05, 0x14, 0x94, 0x08, 0x70, 0xa7, 0x50, 0x48, 0x52, 0x50, 0x02, 0x52, 0x50, 0x31, 0x28, 0xa0, 0x04, 0xa2, 0x80, 0x0a, 0x28, 0x01, 0x28, 0xa0, 0x02, 0x92, 0x80, 0x0a, 0x29, 0x80, 0x52, 0x50, 0x20, 0xa4, 0xa0, 0x02, 0x92, 0x98, 0x05, 0x14, 0x08, 0x86, 0x8a, 0xb2, 0xc2, 0x96, 0x80, 0x16, 0x8a, 0x40, 0x14, 0xb4, 0x00, 0xb4, 0xf1, 0x52, 0x03, 0xc5, 0x2d, 0x48, 0xc5, 0xed, 0x4d, 0x14, 0x08, 0x90, 0x53, 0xe8, 0x24, 0x4a, 0x4a, 0x04, 0x25, 0x25, 0x03, 0x0a, 0x4a, 0x06, 0x14, 0x94, 0x00, 0x51, 0x40, 0x05, 0x25, 0x00, 0x14, 0x50, 0x21, 0x28, 0xa6, 0x31, 0x28, 0xa0, 0x41, 0x49, 0x40, 0x05, 0x25, 0x30, 0x0a, 0x28, 0x11, 0x0d, 0x15, 0x65, 0x8b, 0x45, 0x00, 0x2d, 0x14, 0x80, 0x29, 0x68, 0x18, 0xe1, 0x4e, 0x15, 0x22, 0x1d, 0x4e, 0xa4, 0x31, 0xcd, 0xf7, 0x6a, 0x2a, 0x42, 0x25, 0x4a, 0x92, 0x82, 0x02, 0x92, 0x80, 0x12, 0x9b, 0x40, 0x05, 0x14, 0x0c, 0x29, 0x28, 0x00, 0xa2, 0x80, 0x0a, 0x28, 0x01, 0x29, 0x28, 0x00, 0xa4, 0xa6, 0x01, 0x49, 0x40, 0x05, 0x25, 0x00, 0x14, 0x94, 0xc0, 0x28, 0xa0, 0x44, 0x34, 0x55, 0x96, 0x2d, 0x14, 0x00, 0x52, 0xd2, 0x00, 0xa5, 0xa0, 0x62, 0x8a, 0x78, 0xa9, 0x01, 0xd4, 0xea, 0x42, 0x1c, 0x7e, 0xed, 0x45, 0x48, 0x07, 0x29, 0xc5, 0x4e, 0x28, 0x25, 0x85, 0x25, 0x02, 0x12, 0x92, 0x81, 0x85, 0x14, 0x00, 0x51, 0x40, 0x05, 0x14, 0x00, 0x94, 0x50, 0x21, 0x29, 0x28, 0x18, 0x94, 0x94, 0xc0, 0x28, 0xa4, 0x02, 0x52, 0x53, 0x00, 0xa2, 0x98, 0x84, 0xa2, 0x80, 0x21, 0xa2, 0xac, 0xb1, 0x68, 0xa0, 0x05, 0xa2, 0x90, 0x05, 0x2d, 0x00, 0x2d, 0x3a, 0xa4, 0x63, 0xa9, 0xc2, 0x90, 0x89, 0x1c, 0xe0, 0x0a, 0x8c, 0xd2, 0x10, 0xda, 0x96, 0x36, 0xa0, 0x19, 0x2d, 0x25, 0x04, 0x0d, 0xa4, 0xa0, 0x61, 0x45, 0x03, 0x16, 0x96, 0x81, 0x06, 0x29, 0x71, 0x40, 0x09, 0x8a, 0x4c, 0x50, 0x02, 0x62, 0x9b, 0x40, 0x09, 0x49, 0x4c, 0x04, 0xa2, 0x81, 0x89, 0x49, 0x4c, 0x02, 0x92, 0x81, 0x05, 0x14, 0x01, 0x0d, 0x15, 0x65, 0x8b, 0x45, 0x20, 0x16, 0x8a, 0x00, 0x29, 0x68, 0x01, 0x69, 0x69, 0x00, 0xf1, 0x52, 0x2d, 0x48, 0x04, 0xfc, 0x8a, 0x8d, 0x4e, 0xe5, 0xa0, 0x41, 0x40, 0x34, 0x01, 0x61, 0x4e, 0x45, 0x2d, 0x22, 0x06, 0xd2, 0x50, 0x30, 0xa5, 0xa0, 0x05, 0xa7, 0x50, 0x02, 0xd2, 0xe2, 0x98, 0x83, 0x14, 0xdc, 0x50, 0x03, 0x4d, 0x37, 0x14, 0x80, 0x6d, 0x25, 0x03, 0x12, 0x92, 0x98, 0xc4, 0xa4, 0xa6, 0x20, 0xa4, 0xa0, 0x02, 0x8a, 0x06, 0x43, 0x45, 0x59, 0x42, 0xd1, 0x48, 0x02, 0x96, 0x98, 0x05, 0x2d, 0x20, 0x14, 0x53, 0x85, 0x20, 0x1e, 0x2a, 0x44, 0xeb, 0x50, 0x03, 0x65, 0x6c, 0x49, 0x51, 0xfd, 0xc9, 0x3d, 0x8d, 0x50, 0x87, 0x9a, 0x6d, 0x20, 0x1e, 0x87, 0x15, 0x35, 0x22, 0x58, 0xda, 0x4a, 0x04, 0x14, 0xb4, 0x0c, 0x75, 0x3a, 0x81, 0x0e, 0xc5, 0x3b, 0x15, 0x43, 0x17, 0x6d, 0x37, 0x14, 0x00, 0xd2, 0x29, 0x86, 0x90, 0x88, 0xcd, 0x36, 0x90, 0xc4, 0xa4, 0xa6, 0x02, 0x52, 0x53, 0x00, 0xa4, 0xa4, 0x01, 0x49, 0x4c, 0x64, 0x54, 0x55, 0x94, 0x2d, 0x14, 0x80, 0x28, 0xa0, 0x05, 0xa2, 0x80, 0x14, 0x54, 0x8b, 0x52, 0xc0, 0x7d, 0x49, 0x1f, 0x5a, 0x90, 0x22, 0x9b, 0xef, 0x9a, 0x8f, 0xaa, 0xe3, 0xd2, 0xa9, 0x08, 0x91, 0x4e, 0xe4, 0xa4, 0xa4, 0x01, 0x52, 0xa3, 0x50, 0x26, 0x3a, 0x92, 0x91, 0x21, 0x4b, 0x40, 0xc7, 0x0a, 0x78, 0xa6, 0x22, 0x40, 0x2a, 0x55, 0x4c, 0xd3, 0x19, 0x2f, 0x92, 0x71, 0x50, 0xb2, 0x50, 0x22, 0x22, 0x2a, 0x33, 0x40, 0x11, 0x9a, 0x69, 0xa9, 0x01, 0xb4, 0xda, 0x63, 0x12, 0x8a, 0x00, 0x4a, 0x28, 0x01, 0x29, 0x29, 0x8c, 0x8a, 0x96, 0xac, 0xa0, 0xa2, 0x80, 0x0a, 0x28, 0x01, 0x68, 0xa4, 0x31, 0xc0, 0x54, 0xa2, 0x90, 0x0e, 0xc5, 0x3e, 0x3e, 0xb5, 0x36, 0x0b, 0x10, 0x4d, 0xf7, 0xcd, 0x31, 0x4e, 0x1b, 0x35, 0x42, 0x1f, 0xf7, 0x24, 0xf6, 0x34, 0xf2, 0x28, 0x60, 0x36, 0x95, 0x4e, 0x29, 0x0a, 0xc4, 0xa2, 0x8a, 0x2c, 0x2b, 0x05, 0x28, 0xa2, 0xc3, 0xb0, 0xf1, 0x52, 0x0a, 0x76, 0x15, 0x89, 0x54, 0x55, 0xfb, 0x38, 0xb7, 0xb8, 0x14, 0xec, 0x1c, 0xa7, 0xa0, 0x36, 0x83, 0x64, 0xf6, 0x3f, 0x67, 0x11, 0x28, 0x38, 0xfb, 0xf8, 0xe7, 0x35, 0xc3, 0x6a, 0x96, 0x0d, 0x69, 0x3b, 0x46, 0xdd, 0x45, 0x0e, 0x16, 0x2a, 0x50, 0x32, 0x9d, 0x6a, 0x16, 0x14, 0x58, 0x9b, 0x11, 0x11, 0x4c, 0x22, 0xa6, 0xc1, 0x61, 0x94, 0x94, 0xec, 0x02, 0x52, 0x52, 0x10, 0x94, 0x50, 0x31, 0x29, 0x29, 0x81, 0xff, 0xd9] + +def display(img_path=None): + img_data = None + if img_path is None: + img_data = jpeg_data + else: + with open(img_path, 'rb') as f: + img_data = f.read() + frame = cv2.imdecode(np.asarray(img_data, dtype=np.uint8), cv2.IMREAD_COLOR) + if frame is not None: + while(1): + cv2.imshow('VIDEO', frame) + if cv2.waitKey(1) & 0xFF == ord('q'): + break + else: + print("No frame") + +if __name__ == "__main__": + if len(sys.argv) == 1: + display() + else: + display(sys.argv[1]) diff --git a/components/rtsp_client/python/rtsp_client.py b/components/rtsp_client/python/rtsp_client.py index 7b5b83e..efe58f8 100644 --- a/components/rtsp_client/python/rtsp_client.py +++ b/components/rtsp_client/python/rtsp_client.py @@ -39,31 +39,78 @@ ''' -huffman_table = [ - # 1st table - # Default luminance DC Huffman table - 0xff, 0xc4, 0x00, 0x1f, 0x00, # header - 0x00, 0x01, 0x05, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x06, 0x07, 0x08, - 0x09, 0x0a, 0x0b, - # 2nd table - # Default luminance AC Huffman table - 0xff, 0xc4, 0x00, 0xb5, 0x10, # header - 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01, 0x7d, 0x01, 0x02, - 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, - 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, - 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, - 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, - 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, - 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, - 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2, 0xe3, 0xe4, - 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, +dc_luminance_table = bytearray([ + 0x00, + 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B +]) + +#default luminance AC Huffman table +ac_luminance_table = bytearray([ + 0x00, + 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04, + 0x07, 0x05, 0x04, 0x04, 0x00, 0x01, 0x02, 0x77, + 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, + 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, + 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, + 0xA1, 0xB1, 0xC1, 0x09, 0x23, 0x33, 0x52, 0xF0, + 0x15, 0x62, 0x72, 0xD1, 0x0A, 0x16, 0x24, 0x34, + 0xE1, 0x25, 0xF1, 0x17, 0x18, 0x19, 0x1A, 0x26, + 0x27, 0x28, 0x29, 0x2A, 0x35, 0x36, 0x37, 0x38, + 0x39, 0x3A, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, + 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7A, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, + 0x97, 0x98, 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, + 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, + 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xC2, 0xC3, + 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xD2, + 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, + 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, + 0xEA, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, + 0xF9, 0xFA +]) + +dc_chrominance_table = bytearray([ + 0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B +]) + +ac_chrominance_table = bytearray([ + 0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04, 0x07, 0x05, 0x04, 0x04, 0x00, 0x01, 0x02, 0x77, + 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, + 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, 0xA1, 0xB1, 0xC1, 0x09, 0x23, 0x33, 0x52, 0xF0, + 0x15, 0x62, 0x72, 0xD1, 0x0A, 0x16, 0x24, 0x34, 0xE1, 0x25, 0xF1, 0x17, 0x18, 0x19, 0x1A, 0x26, + 0x27, 0x28, 0x29, 0x2A, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, + 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, + 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xC2, 0xC3, + 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, + 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, + 0xF9, 0xFA +]) +huffman_table = [ + # Huffman table DC (luminance) + 0xff, 0xc4, + 0x00, 0x1f, 0x00, + 0x00, 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + # Huffman table AC (luminance) + 0xff, 0xc4, + 0x00, 0xb5, 0x10, + 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01, 0x7d, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, + # Huffman table DC (chrominance) + 0xff, 0xc4, + 0x00, 0x1f, 0x01, + 0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + # Huffman table AC (chrominance) + 0xff, 0xc4, + 0x00, 0xb5, 0x11, + 0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04, 0x07, 0x05, 0x04, 0x04, 0x00, 0x01, 0x02, 0x77, 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, 0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0, 0x15, 0x62, 0x72, 0xd1, 0x0a, 0x16, 0x24, 0x34, 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, ] class RtpPacket: @@ -165,10 +212,10 @@ def __init__(self, width, height, q0_quantization_table, q1_quantization_table): 0xFF, 0xE0, # APP0 marker 0x00, 0x10, # Length (16 bytes) 0x4A, 0x46, 0x49, 0x46, 0x00, # JFIF identifier - 0x01, 0x02, # JFIF version 1.2 + 0x01, 0x01, # JFIF version 1.1 0x01, # Units: DPI - 0x00, 0x48, # Xdensity: 72 DPI - 0x00, 0x48, # Ydensity: 72 DPI + 0x00, 0x00, # X density (2 bytes) + 0x00, 0x00, # Y density (2 bytes) 0x00, 0x00 # No thumbnail (width 0, height 0) ]) self.data.write(jfif_app0_marker) @@ -183,6 +230,8 @@ def __init__(self, width, height, q0_quantization_table, q1_quantization_table): self.data.write(b'\xFF\xDB\x00\x43\x01') self.data.write(bytearray(q1_quantization_table)) + self.data.write(bytes(huffman_table)) + # Frame header (SOF0) marker sof0_marker = bytearray([ 0xFF, 0xC0, # SOF0 marker @@ -197,8 +246,6 @@ def __init__(self, width, height, q0_quantization_table, q1_quantization_table): ]) self.data.write(sof0_marker) - self.data.write(bytes(huffman_table)) - # Scan header (SOS) marker # marker(0xFFDA), size of SOS (0x000C), num components(0x03), # component specification parameters, From 108be73cec72f02b2209d65620a2dbe669e53bf5 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Sun, 23 Apr 2023 11:32:33 -0500 Subject: [PATCH 09/11] update main to keep trying reconnect --- main/main.cpp | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/main/main.cpp b/main/main.cpp index ecb4cad..68301e8 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -110,11 +110,12 @@ extern "C" void app_main(void) { } static auto start = std::chrono::high_resolution_clock::now(); auto image_data = image->get_data(); - logger.info("Decoding image of size {}", image_data.size()); - logger.info("Decoding image, shape = {} x {}", image->get_width(), image->get_height()); + logger.info("Decoding image of size {} B, shape = {} x {}", + image_data.size(), image->get_width(), image->get_height()); if (jpeg.openRAM((uint8_t*)(image_data.data()), image_data.size(), drawMCUs)) { - logger.debug("Image size: {} x {}, orientation: {}, bpp: {}", jpeg.getWidth(), - jpeg.getHeight(), jpeg.getOrientation(), jpeg.getBpp()); + logger.debug("Image size: {} x {}, orientation: {}, bpp: {}", + jpeg.getWidth(),jpeg.getHeight(), + jpeg.getOrientation(), jpeg.getBpp()); jpeg.setPixelType(RGB565_BIG_ENDIAN); if (!jpeg.decode(0,0,0)) { logger.error("Error decoding"); @@ -149,14 +150,22 @@ extern "C" void app_main(void) { } jpeg_cv.notify_all(); num_frames_received += 1; - } + }, + .log_level = espp::Logger::Verbosity::WARN, }); std::error_code ec; - rtsp_client.connect(ec); - if (ec) { - logger.error("Error connecting to server: {}", ec.message()); - } + + do { + // clear the error code + ec.clear(); + rtsp_client.connect(ec); + if (ec) { + logger.error("Error connecting to server: {}", ec.message()); + logger.info("Retrying in 1s..."); + std::this_thread::sleep_for(1s); + } + } while (ec); rtsp_client.describe(ec); if (ec) { @@ -177,11 +186,13 @@ extern "C" void app_main(void) { while (true) { auto end = std::chrono::high_resolution_clock::now(); float current_time = std::chrono::duration(end-start).count(); - fmt::print("[TM] {}\n", espp::TaskMonitor::get_latest_info()); - fmt::print("[{:.3f}] Received {} frames\n", current_time, num_frames_received); + // fmt::print("[TM] {}\n", espp::TaskMonitor::get_latest_info()); float disp_elapsed = elapsed; if (disp_elapsed > 1) { - fmt::print("[{:.3f}] Framerate: {} FPS\n", current_time, num_frames_displayed / disp_elapsed); + fmt::print("[{:.3f}] Received {} frames, Framerate: {} FPS\n", + current_time, num_frames_received, num_frames_displayed / disp_elapsed); + } else { + fmt::print("[{:.3f}] Received {} frames\n", current_time, num_frames_received); } std::this_thread::sleep_for(1s); } From 8c959016577448021895f5ac416a9410086544f2 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Sun, 23 Apr 2023 11:45:24 -0500 Subject: [PATCH 10/11] feat(main): enforce size * Update espp submodule to latest * Update main display task to use latest api * Update to enforce size of image queue --- components/espp | 2 +- main/main.cpp | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/components/espp b/components/espp index 1278071..850e641 160000 --- a/components/espp +++ b/components/espp @@ -1 +1 @@ -Subproject commit 1278071bf922f4717f5400c5dc241378588ab8ef +Subproject commit 850e64194c076993ff10b4c2e6c0cf613bc357e1 diff --git a/main/main.cpp b/main/main.cpp index 68301e8..c0585b5 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -94,7 +94,7 @@ extern "C" void app_main(void) { std::atomic num_frames_displayed{0}; std::atomic elapsed{0}; - auto display_task_fn = [&jpeg_mutex, &jpeg_cv, &jpeg_frames, &num_frames_displayed, &elapsed](auto& m, auto& cv) { + auto display_task_fn = [&jpeg_mutex, &jpeg_cv, &jpeg_frames, &num_frames_displayed, &elapsed](auto& m, auto& cv) -> bool { // the original (max) image size is 1600x1200, but the S3 BOX has a resolution of 320x240 // wait on the queue until we have an image ready to display static JPEGDEC jpeg; @@ -126,6 +126,8 @@ extern "C" void app_main(void) { auto end = std::chrono::high_resolution_clock::now(); elapsed = std::chrono::duration(end-start).count(); num_frames_displayed += 1; + // signal that we do not want to stop the task + return false; }; // Start the display task logger.info("Starting display task"); @@ -146,6 +148,9 @@ extern "C" void app_main(void) { .on_jpeg_frame = [&jpeg_mutex, &jpeg_cv, &jpeg_frames, &num_frames_received](std::unique_ptr jpeg_frame) { { std::lock_guard lock(jpeg_mutex); + if (jpeg_frames.size() >= MAX_JPEG_FRAMES) { + jpeg_frames.pop_front(); + } jpeg_frames.push_back(std::move(jpeg_frame)); } jpeg_cv.notify_all(); From df885a8f1b8ddde0a7863f2dfc5a7eee329955b1 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Sun, 23 Apr 2023 13:16:38 -0500 Subject: [PATCH 11/11] update how display is initialized for better image display. --- main/lcd.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/lcd.cpp b/main/lcd.cpp index 6fbf1ae..7c1217b 100644 --- a/main/lcd.cpp +++ b/main/lcd.cpp @@ -194,7 +194,7 @@ void lcd_init() { .backlight_on_value = true, .invert_colors = true, .mirror_x = true, - .mirror_y = true, + .mirror_y = false, }); // initialize the display / lvgl using namespace std::chrono_literals;