Fix H264 sps and pps error

platforms/android/aps-sdk/airplay/src/main/cpp/airplay_handler.cpp

@@ -99,8 +99,9 @@ void airplay_handler::on_mirror_stream_codec(
         oss.write((char *)&sc, 0x4);
         uint16_t sps_length = *(uint16_t *)cursor;
         sps_length = ntohs(sps_length);
+        cursor += sizeof(uint16_t);
         oss.write((char *)cursor, sps_length);
-        cursor += sizeof(uint16_t) + sps_length;
+        cursor += sps_length;
       }
 
       // Parse PPS
@@ -109,8 +110,9 @@ void airplay_handler::on_mirror_stream_codec(
         oss.write((char *)&sc, 0x4);
         uint16_t pps_length = *(uint16_t *)cursor;
         pps_length = ntohs(pps_length);
+        cursor += sizeof(uint16_t);
         oss.write((char *)cursor, pps_length);
-        cursor += sizeof(uint16_t) + pps_length;
+        cursor += pps_length;
       }
 
       std::string buffer = oss.str();

platforms/android/aps-sdk/mobile/src/main/java/com/medialab/apserversdk/APSDemoApplication.java

@@ -111,7 +111,7 @@ public void stopAirPlayServer() {
 
     /** Returns whether extension renderers should be used. */
     public boolean useExtensionRenderers() {
-        return "withExtensions".equals(BuildConfig.FLAVOR);
+        return false;
     }
 
     public DownloadManager getDownloadManager() {
@@ -186,7 +186,7 @@ private void createAirPlayServer() {
                 preference
                         .edit()
                         .putString(DEVICE_UNIQUE_ID, config.getMacAddress())
-                        .commit();
+                        .apply();
 
             }
             String name = String.format("APS[%s]", config.getDeviceID());

platforms/android/aps-sdk/mobile/src/main/java/com/medialab/mediasource/APMirroringH264Reader.java

@@ -19,15 +19,13 @@
 import java.util.List;
 
 public class APMirroringH264Reader implements ElementaryStreamReader {
-    private static final int NAL_UNIT_TYPE_SEI = 6; // Supplemental enhancement information
     private static final int NAL_UNIT_TYPE_SPS = 7; // Sequence parameter set
     private static final int NAL_UNIT_TYPE_PPS = 8; // Picture parameter set
 
     private final boolean allowNonIdrKeyframes;
     private final boolean detectAccessUnits;
     private final APMirroringNalUnitTargetBuffer sps;
     private final APMirroringNalUnitTargetBuffer pps;
-    private final APMirroringNalUnitTargetBuffer sei;
     private long totalBytesWritten;
     private final boolean[] prefixFlags;
 
@@ -41,9 +39,6 @@ public class APMirroringH264Reader implements ElementaryStreamReader {
     // Per packet state that gets reset at the start of each packet.
     private long pesTimeUs;
 
-    // Scratch variables to avoid allocations.
-    private final ParsableByteArray seiWrapper;
-
     /**
      * @param allowNonIdrKeyframes Whether to treat samples consisting of non-IDR I slices as
      *     synchronization samples (key-frames).
@@ -56,16 +51,13 @@ public APMirroringH264Reader(boolean allowNonIdrKeyframes, boolean detectAccessU
         prefixFlags = new boolean[3];
         sps = new APMirroringNalUnitTargetBuffer(NAL_UNIT_TYPE_SPS, 128);
         pps = new APMirroringNalUnitTargetBuffer(NAL_UNIT_TYPE_PPS, 128);
-        sei = new APMirroringNalUnitTargetBuffer(NAL_UNIT_TYPE_SEI, 128);
-        seiWrapper = new ParsableByteArray();
     }
 
     @Override
     public void seek() {
         NalUnitUtil.clearPrefixFlags(prefixFlags);
         sps.reset();
         pps.reset();
-        sei.reset();
         sampleReader.reset();
         totalBytesWritten = 0;
     }
@@ -122,7 +114,7 @@ public void consume(ParsableByteArray data) {
             // Indicate the start of the next NAL unit.
             startNalUnit(absolutePosition, nalUnitType, pesTimeUs);
             // Continue scanning the data.
-            offset = nalUnitOffset + 3;
+            offset = nalUnitOffset + APMirroringNalUnitTargetBuffer.START_CODE_LENGTH;
         }
     }
 
@@ -156,8 +148,8 @@ private void endNalUnit(long position, int offset, int discardPadding, long pesT
                     List<byte[]> initializationData = new ArrayList<>();
                     initializationData.add(Arrays.copyOf(sps.nalData, sps.nalLength));
                     initializationData.add(Arrays.copyOf(pps.nalData, pps.nalLength));
-                    NalUnitUtil.SpsData spsData = NalUnitUtil.parseSpsNalUnit(sps.nalData, 3, sps.nalLength);
-                    NalUnitUtil.PpsData ppsData = NalUnitUtil.parsePpsNalUnit(pps.nalData, 3, pps.nalLength);
+                    NalUnitUtil.SpsData spsData = NalUnitUtil.parseSpsNalUnit(sps.nalData, APMirroringNalUnitTargetBuffer.START_CODE_LENGTH, sps.nalLength);
+                    NalUnitUtil.PpsData ppsData = NalUnitUtil.parsePpsNalUnit(pps.nalData, APMirroringNalUnitTargetBuffer.START_CODE_LENGTH, pps.nalLength);
                     output.format(
                             Format.createVideoSampleFormat(
                                     formatId,
@@ -182,11 +174,11 @@ private void endNalUnit(long position, int offset, int discardPadding, long pesT
                     pps.reset();
                 }
             } else if (sps.isCompleted()) {
-                NalUnitUtil.SpsData spsData = NalUnitUtil.parseSpsNalUnit(sps.nalData, 3, sps.nalLength);
+                NalUnitUtil.SpsData spsData = NalUnitUtil.parseSpsNalUnit(sps.nalData, APMirroringNalUnitTargetBuffer.START_CODE_LENGTH, sps.nalLength);
                 sampleReader.putSps(spsData);
                 sps.reset();
             } else if (pps.isCompleted()) {
-                NalUnitUtil.PpsData ppsData = NalUnitUtil.parsePpsNalUnit(pps.nalData, 3, pps.nalLength);
+                NalUnitUtil.PpsData ppsData = NalUnitUtil.parsePpsNalUnit(pps.nalData, APMirroringNalUnitTargetBuffer.START_CODE_LENGTH, pps.nalLength);
                 sampleReader.putPps(ppsData);
                 pps.reset();
             }
@@ -281,7 +273,7 @@ public void startNalUnit(long position, int type, long pesTimeUs) {
         }
 
         /**
-         * Called to pass stream data. The data passed should not include the 3 byte start code.
+         * Called to pass stream data. The data passed should not include the 4 byte start code.
          *
          * @param data Holds the data being passed.
          * @param offset The offset of the data in {@code data}.

platforms/android/aps-sdk/mobile/src/main/java/com/medialab/mediasource/APMirroringNalUnitTargetBuffer.java

@@ -8,6 +8,7 @@
  * encountered in the stream.
  */
 /* package */ final class APMirroringNalUnitTargetBuffer {
+    public static final int START_CODE_LENGTH = 4;
 
     private final int targetType;
 
@@ -21,7 +22,7 @@ public APMirroringNalUnitTargetBuffer(int targetType, int initialCapacity) {
         this.targetType = targetType;
 
         // Initialize data with a start code in the first three bytes.
-        nalData = new byte[3 + initialCapacity];
+        nalData = new byte[START_CODE_LENGTH + initialCapacity];
         nalData[2] = 1;
     }
 
@@ -50,7 +51,7 @@ public void startNalUnit(int type) {
         isFilling = type == targetType;
         if (isFilling) {
             // Skip the three byte start code when writing data.
-            nalLength = 3;
+            nalLength = START_CODE_LENGTH;
             isCompleted = false;
         }
     }

src/service/ap_mirror_stream_service.cpp

@@ -2,14 +2,17 @@
 #include <crypto/ap_crypto.h>
 #include <service/ap_mirror_stream_service.h>
 #include <utils/logger.h>
+#include <sstream>
 
 namespace aps {
 namespace service {
 ap_mirror_stream_connection::ap_mirror_stream_connection(
     asio::io_context &io_ctx, aps::ap_crypto_ptr &crypto,
     aps::ap_handler_ptr handler /*= 0*/)
-    : aps::network::tcp_connection_base(io_ctx), handler_(handler),
-      buffer_(SMS_BUFFER_SIZE), crypto_(crypto) {
+    : aps::network::tcp_connection_base(io_ctx),
+      handler_(handler),
+      buffer_(SMS_BUFFER_SIZE),
+      crypto_(crypto) {
   header_ = (sms_packet_header_t *)buffer_.data();
   payload_ = buffer_.data() + sizeof(sms_packet_header_t);
 
@@ -20,6 +23,9 @@ ap_mirror_stream_connection::ap_mirror_stream_connection(
 ap_mirror_stream_connection::~ap_mirror_stream_connection() {
   LOGD() << "ap_video_stream_session(" << std::hex << this
          << ") is destroying.";
+#if defined(WIN32) && PERSIST_VIDEO_DATA_TO_FILE
+  close_video_data_file();
+#endif
 }
 
 void ap_mirror_stream_connection::start() { post_receive_packet_header(); }
@@ -31,7 +37,7 @@ void ap_mirror_stream_connection::post_receive_packet_header() {
       asio::bind_executor(
           strand_,
           std::bind(&ap_mirror_stream_connection::on_packet_header_received,
-                    shared_from_this(), // Keep the session alive
+                    shared_from_this(),  // Keep the session alive
                     std::placeholders::_1, std::placeholders::_2)));
 }
 
@@ -52,7 +58,7 @@ void ap_mirror_stream_connection::post_receive_packet_payload() {
       asio::bind_executor(
           strand_,
           std::bind(&ap_mirror_stream_connection::on_packet_payload_received,
-                    shared_from_this(), // Keep the session alive
+                    shared_from_this(),  // Keep the session alive
                     std::placeholders::_1, std::placeholders::_2)));
 }
 
@@ -70,18 +76,23 @@ void ap_mirror_stream_connection::on_packet_payload_received(
 }
 
 void ap_mirror_stream_connection::process_packet() {
-  // Convert the stream to 
+  // Convert the stream to
   // 00 00 00 01 SPS  00 00 00 01 PPS  00 00 00 01 NALU
   // 00 00 00 01 NALU 00 00 00 01 NALU 00 00 00 01 ...
 
   if (sms_video_data == header_->payload_type ||
       sms_payload_4096 == header_->payload_type) {
     // Process the video packet
     LOGV() << "mirror VIDEO packet: " << header_->payload_size;
-    // Parse the packet, each packet contains 1 NALU, 
+    // Parse the packet, each packet contains 1 NALU,
     // replace the first 4bytes with 00 00 00 01
     sms_video_data_packet_t *p = (sms_video_data_packet_t *)header_;
     crypto_->decrypt_video_frame(payload_, p->payload_size);
+
+#if defined(WIN32) && PERSIST_VIDEO_DATA_TO_FILE
+    append_nalu(p);
+#endif
+
     if (handler_) {
       handler_->on_mirror_stream_data(p);
     }
@@ -92,21 +103,9 @@ void ap_mirror_stream_connection::process_packet() {
     LOGV() << "mirror CODEC packet: " << header_->payload_size;
     sms_video_codec_packet_t *p = (sms_video_codec_packet_t *)header_;
 
-    //// Parse SPS
-    //uint8_t *cursor = p->start;
-    //for (int i = 0; i < p->sps_count; i++) {
-    //  uint16_t sps_length = *(uint16_t *)cursor;
-    //  sps_length = ntohs(sps_length);
-    //  cursor += sizeof(uint16_t) + sps_length;
-    //}
-
-    //// Parse PPS
-    //uint8_t pps_count = *cursor++;
-    //for (int i = 0; i < pps_count; i++) {
-    //  uint16_t pps_length = *(uint16_t *)cursor;
-    //  pps_length = ntohs(pps_length);
-    //  cursor += sizeof(uint16_t) + pps_length;
-    //}
+#if defined(WIN32) && PERSIST_VIDEO_DATA_TO_FILE
+    init_video_data_file(p);
+#endif
 
     if (handler_) {
       handler_->on_mirror_stream_codec(p);
@@ -123,47 +122,101 @@ void ap_mirror_stream_connection::process_packet() {
 void ap_mirror_stream_connection::handle_socket_error(
     const asio::error_code &e) {
   switch (e.value()) {
-  case asio::error::eof:
-    return;
-
-  case asio::error::connection_reset:
-  case asio::error::connection_aborted:
-  case asio::error::access_denied:
-  case asio::error::address_family_not_supported:
-  case asio::error::address_in_use:
-  case asio::error::already_connected:
-  case asio::error::connection_refused:
-  case asio::error::bad_descriptor:
-  case asio::error::fault:
-  case asio::error::host_unreachable:
-  case asio::error::in_progress:
-  case asio::error::interrupted:
-  case asio::error::invalid_argument:
-  case asio::error::message_size:
-  case asio::error::name_too_long:
-  case asio::error::network_down:
-  case asio::error::network_reset:
-  case asio::error::network_unreachable:
-  case asio::error::no_descriptors:
-  case asio::error::no_buffer_space:
-  case asio::error::no_protocol_option:
-  case asio::error::not_connected:
-  case asio::error::not_socket:
-  case asio::error::operation_not_supported:
-  case asio::error::shut_down:
-  case asio::error::timed_out:
-  case asio::error::would_block:
-    break;
+    case asio::error::eof:
+      return;
+
+    case asio::error::connection_reset:
+    case asio::error::connection_aborted:
+    case asio::error::access_denied:
+    case asio::error::address_family_not_supported:
+    case asio::error::address_in_use:
+    case asio::error::already_connected:
+    case asio::error::connection_refused:
+    case asio::error::bad_descriptor:
+    case asio::error::fault:
+    case asio::error::host_unreachable:
+    case asio::error::in_progress:
+    case asio::error::interrupted:
+    case asio::error::invalid_argument:
+    case asio::error::message_size:
+    case asio::error::name_too_long:
+    case asio::error::network_down:
+    case asio::error::network_reset:
+    case asio::error::network_unreachable:
+    case asio::error::no_descriptors:
+    case asio::error::no_buffer_space:
+    case asio::error::no_protocol_option:
+    case asio::error::not_connected:
+    case asio::error::not_socket:
+    case asio::error::operation_not_supported:
+    case asio::error::shut_down:
+    case asio::error::timed_out:
+    case asio::error::would_block:
+      break;
   }
 
   LOGE() << "Socket error[" << e.value() << "]: " << e.message();
 }
 
+#if defined(WIN32) && defined(PERSIST_VIDEO_DATA_TO_FILE)
+void ap_mirror_stream_connection::init_video_data_file(
+    sms_video_codec_packet_t *p) {
+  if (video_data_file_.is_open()) {
+    video_data_file_.close();
+  }
+
+  auto mode =
+      std::ios_base::binary | std::ios_base::binary | std::ios_base::trunc;
+  video_data_file_.open("ap-mirroring.h264", mode);
+
+  uint32_t sc = htonl(0x01);
+  std::ostringstream oss;
+
+  // Parse SPS
+  uint8_t *cursor = (uint8_t *)p->start;
+  for (int i = 0; i < p->sps_count; i++) {
+    oss.write((char *)&sc, 0x4);
+    uint16_t sps_length = *(uint16_t *)cursor;
+    sps_length = ntohs(sps_length);
+    cursor += sizeof(uint16_t);
+    oss.write((char *)cursor, sps_length);
+    cursor += sps_length;
+  }
+
+  // Parse PPS
+  uint8_t pps_count = *cursor++;
+  for (int i = 0; i < pps_count; i++) {
+    oss.write((char *)&sc, 0x4);
+    uint16_t pps_length = *(uint16_t *)cursor;
+    pps_length = ntohs(pps_length);
+    cursor += sizeof(uint16_t);
+    oss.write((char *)cursor, pps_length);
+    cursor += pps_length;
+  }
+  std::string buffer = oss.str();
+  video_data_file_.write((char *)buffer.c_str(), buffer.length());
+}
+
+void ap_mirror_stream_connection::append_nalu(sms_video_data_packet_t *p) {
+  p->payload[0] = 0;
+  p->payload[1] = 0;
+  p->payload[2] = 0;
+  p->payload[3] = 1;
+  video_data_file_.write((char *)p->payload, p->payload_size);
+}
+
+void ap_mirror_stream_connection::close_video_data_file() {
+  video_data_file_.close();
+}
+
+#endif
+
 ap_mirror_stream_service::ap_mirror_stream_service(aps::ap_crypto_ptr &crypto,
                                                    uint16_t port,
                                                    aps::ap_handler_ptr &handler)
     : aps::network::tcp_service_base("ap_video_stream_service", port, true),
-      handler_(handler), crypto_(crypto) {
+      handler_(handler),
+      crypto_(crypto) {
   bind_thread_actions(
       std::bind(&ap_mirror_stream_service::on_thread_start, this),
       std::bind(&ap_mirror_stream_service::on_thread_stop, this));
@@ -189,5 +242,5 @@ void ap_mirror_stream_service::on_thread_stop() {
   }
 }
 
-} // namespace service
-} // namespace aps
+}  // namespace service
+}  // namespace aps