Support SEI in HLS

src/main/java/io/antmedia/AppSettings.java

@@ -2042,6 +2042,12 @@ public boolean isWriteStatsToDatastore() {
 	 */
 	@Value("${id3TagEnabled:false}")
 	private boolean id3TagEnabled = false;
+
+	/**
+	 * Enables the SEI data for HLS
+	 */
+	@Value("${seiEnabled:false}")
+	private boolean seiEnabled = false;
 
 	/**
 	 * Ant Media Server can get the audio level from incoming RTP Header in WebRTC streaming and send to the viewers.
@@ -3584,6 +3590,14 @@ public void setId3TagEnabled(boolean id3TagEnabled) {
 		this.id3TagEnabled = id3TagEnabled;
 	}
 
+	public boolean isSeiEnabled() {
+		return seiEnabled;
+	}
+
+	public void setSeiEnabled(boolean seiEnabled) {
+		this.seiEnabled = seiEnabled;
+	}
+
 	public boolean isSendAudioLevelToViewers() {
 		return sendAudioLevelToViewers;
 	}

src/main/java/io/antmedia/muxer/HLSMuxer.java

@@ -2,20 +2,18 @@
 
 import static org.bytedeco.ffmpeg.global.avcodec.*;
 import static org.bytedeco.ffmpeg.global.avformat.avformat_alloc_output_context2;
-import static org.bytedeco.ffmpeg.global.avutil.AVMEDIA_TYPE_DATA;
-import static org.bytedeco.ffmpeg.global.avutil.av_rescale_q;
+import static org.bytedeco.ffmpeg.global.avutil.*;
+import static org.bytedeco.ffmpeg.global.avutil.AV_OPT_SEARCH_CHILDREN;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.file.Files;
 
 import org.apache.commons.lang3.StringUtils;
-import org.bytedeco.ffmpeg.avcodec.AVCodec;
-import org.bytedeco.ffmpeg.avcodec.AVCodecContext;
-import org.bytedeco.ffmpeg.avcodec.AVCodecParameters;
-import org.bytedeco.ffmpeg.avcodec.AVPacket;
+import org.bytedeco.ffmpeg.avcodec.*;
 import org.bytedeco.ffmpeg.avformat.AVFormatContext;
+import org.bytedeco.ffmpeg.avformat.AVStream;
 import org.bytedeco.ffmpeg.avutil.AVRational;
 import org.bytedeco.ffmpeg.global.avcodec;
 import org.bytedeco.javacpp.BytePointer;
@@ -28,6 +26,10 @@
 
 public class HLSMuxer extends Muxer  {
 
+	public static final String SEI_USER_DATA = "sei_user_data";
+
+	private static final String SEI_UUID = "086f3693-b7b3-4f2c-9653-21492feee5b8+";
+
 	private static final String SEGMENT_SUFFIX_TS = "%0"+SEGMENT_INDEX_LENGTH+"d.ts";
 
 	protected static Logger logger = LoggerFactory.getLogger(HLSMuxer.class);
@@ -68,6 +70,9 @@ public class HLSMuxer extends Muxer  {
 
 	private boolean id3Enabled = false;
 
+	private boolean seiEnabled = false;
+
+
 	public HLSMuxer(Vertx vertx, StorageClient storageClient, String s3StreamsFolderPath, int uploadExtensionsToS3, String httpEndpoint, boolean addDateTimeToResourceName) {
 		super(vertx);
 		this.storageClient = storageClient;
@@ -351,19 +356,58 @@ public synchronized boolean addStream(AVCodec codec, AVCodecContext codecContext
 			logger.error("Cannot get codec parameters for {}", streamId);
 		}
 
-		//call super directly because no need to add bit stream filter 
+		//call super directly because no need to add bit stream filter
 		return super.addStream(codecParameter, codecContext.time_base(), streamIndex);
 	}
+
+	@Override
+	public synchronized boolean addVideoStream(int width, int height, AVRational timebase, int codecId, int streamIndex,
+											   boolean isAVC, AVCodecParameters codecpar) {
+
+		boolean result = super.addVideoStream(width, height, timebase, codecId, streamIndex, isAVC, codecpar);
+		if (result && seiEnabled)
+		{
+			AVStream outStream = getOutputFormatContext().streams(inputOutputStreamIndexMap.get(streamIndex));
+
+			setBitstreamFilter("h264_metadata");
+
+			AVBSFContext avbsfContext = initVideoBitstreamFilter(getBitStreamFilter(), outStream.codecpar(), inputTimeBaseMap.get(streamIndex));
+
+			if (avbsfContext != null) {
+				int ret = avcodec_parameters_copy(outStream.codecpar(), avbsfContext.par_out());
+				result = ret == 0;
+			}
+
+			setSeiData("initial sei data");
+
+			logger.info("Adding video stream index:{} for stream:", streamIndex);
+		}
+
+		return result;
+	}
+
+	public void setSeiData(String data) {
+		int ret = av_opt_set(bsfFilterContextList.get(0).priv_data(), SEI_USER_DATA, SEI_UUID+data, AV_OPT_SEARCH_CHILDREN);
+		logError(ret, "Cannot set sei_user_data for {} and error is {}", streamId);
+
+
+		ret = av_bsf_init(bsfFilterContextList.get(0));
+		logError(ret, "Cannot update sei_user_data for {} and error is {}", streamId);
+
+	}
+
+	public static void logError(int ret, String message, String streamId) {
+		if (ret < 0 && logger.isErrorEnabled()) {
+			logger.error(message, streamId, Muxer.getErrorDefinition(ret));
+		}
+	}
 
 
 	@Override
 	public synchronized boolean addStream(AVCodecParameters codecParameters, AVRational timebase, int streamIndex) 
 	{
-		bsfVideoName = "h264_mp4toannexb";
-
-		boolean ret = super.addStream(codecParameters, timebase, streamIndex);
-
-		return ret;
+		setBitstreamFilter("h264_mp4toannexb");
+		return super.addStream(codecParameters, timebase, streamIndex);
 	}
 
 	public boolean addID3Stream() {
@@ -419,7 +463,12 @@ public String getSegmentFilename() {
 	public void setId3Enabled(boolean id3Enabled) {
 		this.id3Enabled = id3Enabled;
 	}
-
+
+	public void setSeiEnabled(boolean seiEnabled) {
+		this.seiEnabled = seiEnabled;
+	}
+
+
 	@Override
 	protected synchronized void clearResource() {
 		super.clearResource();

src/main/java/io/antmedia/muxer/Mp4Muxer.java

@@ -143,8 +143,8 @@ public synchronized boolean addVideoStream(int width, int height, AVRational tim
 	 */
 	@Override
 	protected boolean prepareAudioOutStream(AVStream inStream, AVStream outStream) {
-		if (bsfVideoName != null) {
-			AVBitStreamFilter adtsToAscBsf = av_bsf_get_by_name(this.bsfVideoName);
+		if (getBitStreamFilter() != null) {
+			AVBitStreamFilter adtsToAscBsf = av_bsf_get_by_name(this.getBitStreamFilter());
 			bsfContext = new AVBSFContext(null);
 
 			int ret = av_bsf_alloc(adtsToAscBsf, bsfContext);

src/main/java/io/antmedia/muxer/MuxAdaptor.java

@@ -193,6 +193,16 @@ public boolean addID3Data(String data) {
 		return false;
 	}
 
+	public boolean addSEIData(String data) {
+		for (Muxer muxer : muxerList) {
+			if(muxer instanceof HLSMuxer) {
+				((HLSMuxer)muxer).setSeiData(data);
+				return true;
+			}
+		}
+		return false;
+	}
+
 	public static class PacketTime {
 		public final long packetTimeMs;
 		public final long systemTimeMs;
@@ -433,6 +443,7 @@ public boolean init(IScope scope, String streamId, boolean isAppend) {
 			hlsMuxer.setHlsParameters( hlsListSize, hlsTime, hlsPlayListType, getAppSettings().getHlsflags(), getAppSettings().getHlsEncryptionKeyInfoFile(), getAppSettings().getHlsSegmentType());
 			hlsMuxer.setDeleteFileOnExit(deleteHLSFilesOnExit);
 			hlsMuxer.setId3Enabled(appSettings.isId3TagEnabled());
+			hlsMuxer.setSeiEnabled(appSettings.isSeiEnabled());
 			addMuxer(hlsMuxer);
 			logger.info("adding HLS Muxer for {}", streamId);
 		}

src/main/java/io/antmedia/muxer/Muxer.java

@@ -12,18 +12,8 @@
 import static org.bytedeco.ffmpeg.global.avformat.avformat_open_input;
 import static org.bytedeco.ffmpeg.global.avformat.avformat_write_header;
 import static org.bytedeco.ffmpeg.global.avformat.avio_closep;
-import static org.bytedeco.ffmpeg.global.avutil.AVMEDIA_TYPE_AUDIO;
-import static org.bytedeco.ffmpeg.global.avutil.AVMEDIA_TYPE_DATA;
-import static org.bytedeco.ffmpeg.global.avutil.AVMEDIA_TYPE_VIDEO;
-import static org.bytedeco.ffmpeg.global.avutil.AV_NOPTS_VALUE;
-import static org.bytedeco.ffmpeg.global.avutil.AV_PIX_FMT_YUV420P;
-import static org.bytedeco.ffmpeg.global.avutil.AV_ROUND_NEAR_INF;
-import static org.bytedeco.ffmpeg.global.avutil.AV_ROUND_PASS_MINMAX;
-import static org.bytedeco.ffmpeg.global.avutil.av_dict_free;
-import static org.bytedeco.ffmpeg.global.avutil.av_dict_set;
-import static org.bytedeco.ffmpeg.global.avutil.av_rescale_q;
-import static org.bytedeco.ffmpeg.global.avutil.av_rescale_q_rnd;
-import static org.bytedeco.ffmpeg.global.avutil.av_strerror;
+import static org.bytedeco.ffmpeg.global.avutil.*;
+import static org.bytedeco.ffmpeg.global.avutil.AV_OPT_SEARCH_CHILDREN;
 
 import java.io.File;
 import java.io.IOException;
@@ -38,8 +28,6 @@
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import io.antmedia.FFmpegUtilities;
 import io.antmedia.rest.RestServiceBase;
@@ -133,14 +121,14 @@ public abstract class Muxer {
 	/**
 	 * Bitstream filter name that will be applied to packets
 	 */
-	protected String bsfVideoName = null;
+	protected List<String> bsfVideoNames = new ArrayList<>();
 
 	protected String streamId = null;
 
 	protected Map<Integer, AVRational> inputTimeBaseMap = new ConcurrentHashMap<>();
 
 
-	protected AVBSFContext videoBsfFilterContext = null;
+	protected List<AVBSFContext> bsfFilterContextList = new ArrayList<>();
 
 	protected int videoWidth;
 	protected int videoHeight;
@@ -464,10 +452,10 @@ protected synchronized void clearResource() {
 			audioPkt = null;
 		}
 
-		if (videoBsfFilterContext != null) {
+		for (AVBSFContext videoBsfFilterContext: bsfFilterContextList) {
 			av_bsf_free(videoBsfFilterContext);
-			videoBsfFilterContext = null;
 		}
+		bsfFilterContextList.clear();
 
 		/* close output */
 		if (outputFormatContext != null &&
@@ -532,7 +520,7 @@ public void logPacketIssue(String format, Object... arguments) {
 	/**
 	 * Write packets to the output. This function is used in transcoding.
 	 * Previously, It's the replacement of {link {@link #writePacket(AVPacket)}
-	 * @param avpacket
+	 * @param pkt
 	 * @param codecContext
 	 */
 	public synchronized void writePacket(AVPacket pkt, AVCodecContext codecContext) {
@@ -569,11 +557,15 @@ public ByteBuffer getPacketBufferWithExtradata(byte[] extradata, AVPacket pkt){
 
 
 	public void setBitstreamFilter(String bsfName) {
-		this.bsfVideoName = bsfName;
+		bsfVideoNames.add(bsfName);
 	}
 
 	public String getBitStreamFilter() {
-		return bsfVideoName;
+		if(!bsfVideoNames.isEmpty())
+		{
+			return bsfVideoNames.get(0);
+		}
+		return null;
 	}
 
 	public File getFile() {
@@ -836,16 +828,18 @@ && isCodecSupported(codecParameters.codec_id()) &&
 			//if it's not running add to the list
 			registeredStreamIndexList.add(streamIndex);
 
-			if (bsfVideoName != null && codecParameters.codec_type() == AVMEDIA_TYPE_VIDEO) 
+			if (codecParameters.codec_type() == AVMEDIA_TYPE_VIDEO)
 			{
-				AVBSFContext videoBitstreamFilter = initVideoBitstreamFilter(codecParameters, timebase);
-				if (videoBitstreamFilter != null) 
-				{
-					codecParameters = videoBitstreamFilter.par_out();
-					timebase = videoBitstreamFilter.time_base_out();
+				for (String bsfVideoName: bsfVideoNames) {
+					AVBSFContext videoBitstreamFilter = initVideoBitstreamFilter(bsfVideoName, codecParameters, timebase);
+					if (videoBitstreamFilter != null)
+					{
+						codecParameters = videoBitstreamFilter.par_out();
+						timebase = videoBitstreamFilter.time_base_out();
+					}
 				}
-
 			}
+
 			String codecType = "audio";
 			if (codecParameters.codec_type() == AVMEDIA_TYPE_VIDEO) 
 			{
@@ -886,9 +880,9 @@ else if (codecParameters.codec_type() == AVMEDIA_TYPE_DATA)
 		return result;
 	}
 
-	public AVBSFContext initVideoBitstreamFilter(AVCodecParameters codecParameters, AVRational timebase) {
+	public AVBSFContext initVideoBitstreamFilter(String bsfVideoName, AVCodecParameters codecParameters, AVRational timebase) {
 		AVBitStreamFilter bsfilter = av_bsf_get_by_name(bsfVideoName);
-		videoBsfFilterContext = new AVBSFContext(null);
+		AVBSFContext videoBsfFilterContext = new AVBSFContext(null);
 		int ret = av_bsf_alloc(bsfilter, videoBsfFilterContext);
 
 		if (ret < 0) {
@@ -909,6 +903,8 @@ public AVBSFContext initVideoBitstreamFilter(AVCodecParameters codecParameters,
 			return null;
 		}
 
+		bsfFilterContextList.add(videoBsfFilterContext);
+
 		return videoBsfFilterContext;
 	}
 
@@ -1180,32 +1176,22 @@ public void addExtradataIfRequired(AVPacket pkt, boolean isKeyFrame)
 	protected void writeVideoFrame(AVPacket pkt, AVFormatContext context) {
 		int ret;
 
-
-		if (videoBsfFilterContext != null) 
+		for(AVBSFContext videoBsfFilterContext : bsfFilterContextList)
 		{
 			ret = av_bsf_send_packet(videoBsfFilterContext, pkt);
 			if (ret < 0) {
 				logger.warn("Cannot send packet to bit stream filter for stream:{}", streamId);
 				return;
 			}
-			while (av_bsf_receive_packet(videoBsfFilterContext, pkt) == 0)
-			{
-				logger.trace("write video packet pts:{} dts:{}", pkt.pts(), pkt.dts());
-				ret = av_write_frame(context, tmpPacket);
-				if (ret < 0 && logger.isWarnEnabled()) {
-					logger.warn("cannot write video frame to muxer({}) av_bsf_receive_packet. Error is {} ", file.getName(), getErrorDefinition(ret));
-				}
-			}
+			ret = av_bsf_receive_packet(videoBsfFilterContext, pkt);
 		}
-		else 
-		{
-			logger.trace("write video packet pts:{} dts:{}", pkt.pts(), pkt.dts());
-			ret = av_write_frame(context, pkt);
-			if (ret < 0 && logger.isWarnEnabled()) {
-				//TODO: this is written for some muxers like HLS because normalized video time is coming from WebRTC
-				//WebRTCVideoForwarder#getVideoTime. Fix this problem when upgrading the webrtc stack
-				logger.warn("cannot write video frame to muxer({}). Pts: {} dts:{}  Error is {} ", file.getName(), pkt.pts(), pkt.dts(), getErrorDefinition(ret));
-			}
+
+		logger.trace("write video packet pts:{} dts:{}", pkt.pts(), pkt.dts());
+		ret = av_write_frame(context, pkt);
+		if (ret < 0 && logger.isWarnEnabled()) {
+			//TODO: this is written for some muxers like HLS because normalized video time is coming from WebRTC
+			//WebRTCVideoForwarder#getVideoTime. Fix this problem when upgrading the webrtc stack
+			logger.warn("cannot write video frame to muxer({}). Pts: {} dts:{}  Error is {} ", file.getName(), pkt.pts(), pkt.dts(), getErrorDefinition(ret));
 		}
 	}