Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Try h265 #615

Merged
merged 4 commits into from
Aug 20, 2023
Merged

Try h265 #615

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
165377a
first try of h265
mpromonet Aug 19, 2023
4c989c5
rework
mpromonet Aug 20, 2023
f618b62
forward frametype
mpromonet Aug 20, 2023
1a88adc
send sps even decoding fails
mpromonet Aug 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 13 additions & 12 deletions inc/EncodedVideoFrameBuffer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,29 +17,30 @@
class EncodedVideoI420Buffer : public webrtc::I420BufferInterface
{
public:
EncodedVideoI420Buffer(int width, int height, const rtc::scoped_refptr<webrtc::EncodedImageBufferInterface> &encoded_data) : width_(width), height_(height), encoded_data_(encoded_data)
{
}
virtual int width() const { return width_; }
virtual int height() const { return height_; }
virtual const uint8_t *DataY() const { return encoded_data_->data(); }
EncodedVideoI420Buffer(int width, int height, const rtc::scoped_refptr<webrtc::EncodedImageBufferInterface> &encoded_data, webrtc::VideoFrameType frameType)
: m_width(width), m_height(height), m_encoded_data(encoded_data), m_frameType(frameType) {}
virtual int width() const { return m_width; }
virtual int height() const { return m_height; }
virtual const uint8_t *DataY() const { return m_encoded_data->data(); }
virtual const uint8_t *DataU() const { return NULL; }
virtual const uint8_t *DataV() const { return NULL; }
virtual int StrideY() const { return encoded_data_->size(); }
virtual int StrideY() const { return m_encoded_data->size(); }
virtual int StrideU() const { return 0; }
virtual int StrideV() const { return 0; }
webrtc::VideoFrameType getFrameType() const { return m_frameType; }

private:
const int width_;
const int height_;
rtc::scoped_refptr<webrtc::EncodedImageBufferInterface> encoded_data_;
const int m_width;
const int m_height;
rtc::scoped_refptr<webrtc::EncodedImageBufferInterface> m_encoded_data;
webrtc::VideoFrameType m_frameType;
};

class EncodedVideoFrameBuffer : public webrtc::VideoFrameBuffer
{
public:
EncodedVideoFrameBuffer(int width, int height, const rtc::scoped_refptr<webrtc::EncodedImageBufferInterface> &encoded_data)
: buffer_(new rtc::RefCountedObject<EncodedVideoI420Buffer>(width, height, encoded_data)) {}
EncodedVideoFrameBuffer(int width, int height, const rtc::scoped_refptr<webrtc::EncodedImageBufferInterface> &encoded_data, webrtc::VideoFrameType frameType)
: buffer_(new rtc::RefCountedObject<EncodedVideoI420Buffer>(width, height, encoded_data, frameType)) {}
virtual Type type() const { return webrtc::VideoFrameBuffer::Type::kNative; }
virtual rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() { return webrtc::I420Buffer::Create(width(), height()); }
virtual int width() const { return buffer_->width(); }
Expand Down
16 changes: 10 additions & 6 deletions inc/NullDecoder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,17 @@ class NullDecoder : public webrtc::VideoDecoder {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
rtc::scoped_refptr<webrtc::EncodedImageBufferInterface> encodedData = input_image.GetEncodedData();
rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer = rtc::make_ref_counted<EncodedVideoFrameBuffer>(m_settings.max_render_resolution().Width(), m_settings.max_render_resolution().Height(), encodedData);
rtc::scoped_refptr<webrtc::VideoFrameBuffer> frameBuffer = rtc::make_ref_counted<EncodedVideoFrameBuffer>(m_settings.max_render_resolution().Width(), m_settings.max_render_resolution().Height(), encodedData, input_image._frameType);

webrtc::VideoFrame frame(buffer, webrtc::kVideoRotation_0, render_time_ms * rtc::kNumMicrosecsPerMillisec);
frame.set_timestamp(input_image.Timestamp());
frame.set_ntp_time_ms(input_image.NtpTimeMs());

RTC_LOG(LS_VERBOSE) << "Decode " << frame.id() << " " << input_image._frameType << " " << buffer->width() << "x" << buffer->height() << " " << buffer->GetI420()->StrideY();
webrtc::VideoFrame frame = webrtc::VideoFrame::Builder()
.set_video_frame_buffer(frameBuffer)
.set_rotation(webrtc::kVideoRotation_0)
.set_timestamp_rtp(input_image.Timestamp())
.set_timestamp_ms(render_time_ms)
.set_ntp_time_ms(input_image.NtpTimeMs())
.build();

RTC_LOG(LS_VERBOSE) << "Decode " << frame.id() << " " << input_image._frameType << " " << frameBuffer->width() << "x" << frameBuffer->height() << " " << frameBuffer->GetI420()->StrideY();

m_decoded_image_callback->Decoded(frame);

Expand Down
19 changes: 7 additions & 12 deletions inc/NullEncoder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,17 +48,10 @@ class NullEncoder : public webrtc::VideoEncoder {
}

// compute frametype
uint8_t* data = (uint8_t*)buffer->GetI420()->DataY();
size_t dataSize = buffer->GetI420()->StrideY();
webrtc::VideoFrameType frameType = webrtc::VideoFrameType::kVideoFrameDelta;
std::vector<webrtc::H264::NaluIndex> naluIndexes = webrtc::H264::FindNaluIndices(data, dataSize);
for (webrtc::H264::NaluIndex index : naluIndexes) {
webrtc::H264::NaluType nalu_type = webrtc::H264::ParseNaluType(data[index.payload_start_offset]);
if (nalu_type == webrtc::H264::NaluType::kIdr) {
frameType = webrtc::VideoFrameType::kVideoFrameKey;
break;
}
}
EncodedVideoI420Buffer* encodedBuffer = (EncodedVideoI420Buffer*)buffer->GetI420();
const uint8_t* data = encodedBuffer->DataY();
size_t dataSize = encodedBuffer->StrideY();
webrtc::VideoFrameType frameType = encodedBuffer->getFrameType();

// build webrtc::EncodedImage
webrtc::EncodedImage encoded_image;
Expand All @@ -71,7 +64,9 @@ class NullEncoder : public webrtc::VideoEncoder {

// forward to callback
webrtc::CodecSpecificInfo codec_specific;
codec_specific.codecType = webrtc::VideoCodecType::kVideoCodecH264;
if (m_format.name == "H264") {
codec_specific.codecType = webrtc::VideoCodecType::kVideoCodecH264;
}
webrtc::EncodedImageCallback::Result result = m_encoded_image_callback->OnEncodedImage(encoded_image, &codec_specific);
if (result.error == webrtc::EncodedImageCallback::Result::ERROR_SEND_FAILED) {
RTC_LOG(LS_ERROR) << "Error in parsing EncodedImage" << encoded_image._frameType;
Expand Down
3 changes: 2 additions & 1 deletion inc/V4l2Capturer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,8 @@ class V4l2Capturer : public VideoSource
}

int64_t ts = std::chrono::high_resolution_clock::now().time_since_epoch().count()/1000/1000;
rtc::scoped_refptr<webrtc::VideoFrameBuffer> frameBuffer = rtc::make_ref_counted<EncodedVideoFrameBuffer>(m_capture->getWidth(), m_capture->getHeight(), encodedData);
webrtc::VideoFrameType frameType = idr ? webrtc::VideoFrameType::kVideoFrameKey : webrtc::VideoFrameType::kVideoFrameDelta;
rtc::scoped_refptr<webrtc::VideoFrameBuffer> frameBuffer = rtc::make_ref_counted<EncodedVideoFrameBuffer>(m_capture->getWidth(), m_capture->getHeight(), encodedData, frameType);
webrtc::VideoFrame frame = webrtc::VideoFrame::Builder()
.set_video_frame_buffer(frameBuffer)
.set_rotation(webrtc::kVideoRotation_0)
Expand Down
89 changes: 64 additions & 25 deletions inc/VideoDecoder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,30 +15,26 @@
#include "api/video/i420_buffer.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "modules/video_coding/h264_sprop_parameter_sets.h"
#include "rtc_base/third_party/base64/base64.h"

#include "SessionSink.h"
#include "VideoScaler.h"

#define FOURCC(a, b, c, d) \
((static_cast<uint32_t>(a)) | (static_cast<uint32_t>(b) << 8) | \
(static_cast<uint32_t>(c) << 16) | (static_cast<uint32_t>(d) << 24))

const uint32_t FOURCC_VP9 = FOURCC('V','P','9', 0);
#undef FOURCC

class VideoDecoder : public rtc::VideoSourceInterface<webrtc::VideoFrame>, public webrtc::DecodedImageCallback {
private:
class Frame
{
public:
Frame(): m_timestamp_ms(0) {}
Frame(const rtc::scoped_refptr<webrtc::EncodedImageBuffer> & content, uint64_t timestamp_ms, webrtc::VideoFrameType frameType) : m_content(content), m_timestamp_ms(timestamp_ms), m_frameType(frameType) {}
Frame(const cricket::VideoFormat & format) : m_format(format) {}
Frame(const std::string & format, int width, int height) : m_format(format), m_width(width), m_height(height) {}

rtc::scoped_refptr<webrtc::EncodedImageBuffer> m_content;
uint64_t m_timestamp_ms;
webrtc::VideoFrameType m_frameType;
cricket::VideoFormat m_format;
uint64_t m_timestamp_ms;
webrtc::VideoFrameType m_frameType;
std::string m_format;
int m_width;
int m_height;
};

public:
Expand All @@ -59,6 +55,22 @@ class VideoDecoder : public rtc::VideoSourceInterface<webrtc::VideoFrame>, publi
int width() { return m_scaler.width(); }
int height() { return m_scaler.height(); }

static std::vector<uint8_t> extractParameters(const std::string & buffer)
{
std::vector<uint8_t> binary;
std::string value(buffer);
size_t pos = value.find_first_of(" ;\r\n");
if (pos != std::string::npos)
{
value.erase(pos);
}
rtc::Base64::DecodeFromArray(value.data(), value.size(), rtc::Base64::DO_STRICT, &binary, nullptr);
binary.insert(binary.begin(), H26X_marker, H26X_marker+sizeof(H26X_marker));

return binary;
}


std::vector< std::vector<uint8_t> > getInitFrames(const std::string & codec, const char* sdp) {
std::vector< std::vector<uint8_t> > frames;

Expand Down Expand Up @@ -91,13 +103,33 @@ class VideoDecoder : public rtc::VideoSourceInterface<webrtc::VideoFrame>, publi
RTC_LOG(LS_WARNING) << "Cannot decode SPS:" << sprop;
}
}
} else if (codec == "H265") {
const char* pattern="sprop-vps=";
const char* spropvps=strstr(sdp, pattern);
const char* spropsps=strstr(sdp, "sprop-sps=");
const char* sproppps=strstr(sdp, "sprop-pps=");
if (spropvps && spropsps && sproppps)
{
std::string vpsstr(spropvps+strlen(pattern));
std::vector<uint8_t> vps = extractParameters(vpsstr);
frames.push_back(vps);

std::string spsstr(spropsps+strlen(pattern));
std::vector<uint8_t> sps = extractParameters(spsstr);
frames.push_back(sps);

std::string ppsstr(spropvps+strlen(pattern));
std::vector<uint8_t> pps = extractParameters(ppsstr);
frames.push_back(pps);
}
}


return frames;
}

void postFormat(const cricket::VideoFormat & format) {
Frame frame(format);
void postFormat(const std::string & format, int width, int height) {
Frame frame(format, width, height);
{
std::unique_lock<std::mutex> lock(m_queuemutex);
m_queue.push(frame);
Expand Down Expand Up @@ -157,14 +189,17 @@ class VideoDecoder : public rtc::VideoSourceInterface<webrtc::VideoFrame>, publi
return (m_decoder.get() != NULL);
}

void createDecoder(const cricket::VideoFormat & format) {
void createDecoder(const std::string & format, int width, int height) {
webrtc::VideoDecoder::Settings settings;
webrtc::RenderResolution resolution(format.width, format.height);
webrtc::RenderResolution resolution(width, height);
settings.set_max_render_resolution(resolution);
if (format.fourcc == cricket::FOURCC_H264) {
if (format == "H264") {
m_decoder=m_factory->CreateVideoDecoder(webrtc::SdpVideoFormat(cricket::kH264CodecName));
settings.set_codec_type(webrtc::VideoCodecType::kVideoCodecH264);
} else if (format.fourcc == FOURCC_VP9) {
} else if (format == "H265") {
m_decoder=m_factory->CreateVideoDecoder(webrtc::SdpVideoFormat(format));
settings.set_codec_type(webrtc::VideoCodecType::kVideoCodecGeneric);
} else if (format == "VP9") {
m_decoder=m_factory->CreateVideoDecoder(webrtc::SdpVideoFormat(cricket::kVp9CodecName));
settings.set_codec_type(webrtc::VideoCodecType::kVideoCodecVP9);
}
Expand All @@ -191,21 +226,22 @@ class VideoDecoder : public rtc::VideoSourceInterface<webrtc::VideoFrame>, publi
while (!m_stop) {
Frame frame = this->getFrame();

if (frame.m_format.fourcc != 0) {
cricket::VideoFormat & format = frame.m_format;
if (!frame.m_format.empty()) {

if (this->hasDecoder()) {
if ((m_format.width != format.width) || (m_format.height != format.height)) {
RTC_LOG(LS_INFO) << "format changed => set format from " << m_format.ToString() << " to " << format.ToString();
if ((m_format != frame.m_format) || (m_width != frame.m_width) || (m_height != frame.m_height)) {
RTC_LOG(LS_INFO) << "format changed => set format from " << m_format << " " << m_width << "x" << m_height << " to " << frame.m_format << " " << frame.m_width << "x" << frame.m_height;
m_decoder.reset(NULL);
}
}

if (!this->hasDecoder()) {
RTC_LOG(LS_INFO) << "VideoDecoder:DecoderThread set format:" << format.ToString();
m_format = format;
RTC_LOG(LS_INFO) << "VideoDecoder:DecoderThread set format:" << frame.m_format << " " << frame.m_width << "x" << frame.m_height;
m_format = frame.m_format;
m_width = frame.m_width;
m_height = frame.m_height;

this->createDecoder(format);
this->createDecoder(frame.m_format, frame.m_width, frame.m_height);
}
}

Expand Down Expand Up @@ -259,7 +295,10 @@ class VideoDecoder : public rtc::VideoSourceInterface<webrtc::VideoFrame>, publi
protected:
VideoScaler m_scaler;
std::unique_ptr<webrtc::VideoDecoderFactory>& m_factory;
cricket::VideoFormat m_format;

std::string m_format;
int m_width;
int m_height;

std::queue<Frame> m_queue;
std::mutex m_queuemutex;
Expand Down
2 changes: 1 addition & 1 deletion inc/VideoDecoderFactory.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

class VideoDecoderFactory : public webrtc::VideoDecoderFactory {
public:
VideoDecoderFactory(): supported_formats_({webrtc::SdpVideoFormat(cricket::kH264CodecName)}) {}
VideoDecoderFactory(): supported_formats_({webrtc::SdpVideoFormat(cricket::kH264CodecName),webrtc::SdpVideoFormat("H265")}) {}
virtual ~VideoDecoderFactory() override {}

std::unique_ptr<webrtc::VideoDecoder> CreateVideoDecoder(const webrtc::SdpVideoFormat& format) override {
Expand Down
2 changes: 1 addition & 1 deletion inc/VideoEncoderFactory.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

class VideoEncoderFactory : public webrtc::VideoEncoderFactory {
public:
VideoEncoderFactory(): supported_formats_({webrtc::SdpVideoFormat(cricket::kH264CodecName)}) {}
VideoEncoderFactory(): supported_formats_({webrtc::SdpVideoFormat(cricket::kH264CodecName),webrtc::SdpVideoFormat("H265")}) {}
virtual ~VideoEncoderFactory() override {}

std::unique_ptr<webrtc::VideoEncoder> CreateVideoEncoder(const webrtc::SdpVideoFormat& format) override {
Expand Down
Loading
Loading