Skip to content

Commit

Permalink
Perform async rendering in the main thread on Mac & iOS (#3166)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sauwming
sauwming committed Jul 7, 2022
1 parent b9db6ac commit 7231146
Show file tree
Hide file tree
Showing 3 changed files with 96 additions and 30 deletions.
74 changes: 49 additions & 25 deletions pjmedia/src/pjmedia-videodev/darwin_dev.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,8 @@ @interface VOutDelegate: NSObject
AVCaptureVideoPreviewLayer *prev_layer;

#if TARGET_OS_IPHONE
pj_bool_t is_running;
pj_bool_t is_rendering;
void *render_buf;
pj_size_t render_buf_size;
CGDataProviderRef render_data_provider;
Expand Down Expand Up @@ -512,9 +514,10 @@ static pj_status_t darwin_factory_default_param(pj_pool_t *pool,
@implementation VOutDelegate
#if TARGET_OS_IPHONE
- (void)update_image
{
{
NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();

CGImageRef cgIm = CGImageCreate(stream->size.w, stream->size.h,
8, 32, stream->bytes_per_row, colorSpace,
kCGImageAlphaFirst |
Expand All @@ -527,7 +530,16 @@ - (void)update_image
CGImageRelease(cgIm);

[pool release];
stream->is_rendering = PJ_FALSE;
}

- (void)finish_render
{
/* Do nothing. This function is serialized in the main thread, so when
* it is called, we can be sure that update_image() has completed.
*/
}

#endif

- (void)session_runtime_error:(NSNotification *)notification
Expand Down Expand Up @@ -1311,14 +1323,14 @@ static pj_status_t darwin_stream_start(pjmedia_vid_dev_stream *strm)
{
struct darwin_stream *stream = (struct darwin_stream*)strm;

PJ_UNUSED_ARG(stream);

PJ_LOG(4, (THIS_FILE, "Starting Darwin video stream"));

stream->is_running = PJ_TRUE;

if (stream->cap_session) {
dispatch_sync_on_main_queue(^{
[stream->cap_session startRunning];
});
[stream->cap_session
performSelectorOnMainThread:@selector(startRunning)
withObject:nil waitUntilDone:YES];

if (![stream->cap_session isRunning]) {
/* More info about the error should be reported in
Expand Down Expand Up @@ -1346,16 +1358,23 @@ static pj_status_t darwin_stream_put_frame(pjmedia_vid_dev_stream *strm,
*/
if (frame->size==0 || frame->buf==NULL)
return PJ_SUCCESS;


if (!stream->is_running)
return PJ_EINVALIDOP;

/* Prevent more than one async rendering task. */
if (stream->is_rendering)
return PJ_EIGNORED;

if (stream->frame_size >= frame->size)
pj_memcpy(stream->render_buf, frame->buf, frame->size);
else
pj_memcpy(stream->render_buf, frame->buf, stream->frame_size);

/* Perform video display in a background thread */
dispatch_sync_on_main_queue(^{
[stream->vout_delegate update_image];
});
/* Perform video display in the main thread */
stream->is_rendering = PJ_TRUE;
[stream->vout_delegate performSelectorOnMainThread:@selector(update_image)
withObject:nil waitUntilDone:NO];
#endif

return PJ_SUCCESS;
Expand All @@ -1371,10 +1390,16 @@ static pj_status_t darwin_stream_stop(pjmedia_vid_dev_stream *strm)

PJ_LOG(4, (THIS_FILE, "Stopping Darwin video stream"));

dispatch_sync_on_main_queue(^{
[stream->cap_session stopRunning];
});
[stream->cap_session performSelectorOnMainThread:@selector(stopRunning)
withObject:nil waitUntilDone:YES];
stream->has_image = PJ_FALSE;
stream->is_running = PJ_FALSE;

#if TARGET_OS_IPHONE
/* Wait until the rendering finishes */
[stream->vout_delegate performSelectorOnMainThread:@selector(finish_render)
withObject:nil waitUntilDone:YES];
#endif

return PJ_SUCCESS;
}
Expand Down Expand Up @@ -1411,27 +1436,26 @@ static pj_status_t darwin_stream_destroy(pjmedia_vid_dev_stream *strm)

#if TARGET_OS_IPHONE
if (stream->prev_layer) {
CALayer *prev_layer = stream->prev_layer;
dispatch_sync_on_main_queue(^{
[prev_layer removeFromSuperlayer];
[prev_layer release];
});
[stream->prev_layer
performSelectorOnMainThread:@selector(removeFromSuperlayer)
withObject:nil waitUntilDone:YES];
[stream->prev_layer release];
stream->prev_layer = nil;
}

if (stream->render_view) {
UIView *view = stream->render_view;
dispatch_sync_on_main_queue(^{
[view removeFromSuperview];
[view release];
});
[stream->render_view
performSelectorOnMainThread:@selector(removeFromSuperview)
withObject:nil waitUntilDone:YES];

[stream->render_view release];
stream->render_view = nil;
}

if (stream->render_data_provider) {
CGDataProviderRelease(stream->render_data_provider);
stream->render_data_provider = nil;
}
}
#endif /* TARGET_OS_IPHONE */

if (stream->queue) {
Expand Down
51 changes: 46 additions & 5 deletions pjmedia/src/pjmedia-videodev/ios_opengl_dev.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,11 @@ @interface GLView : UIView
pj_timestamp frame_ts;
unsigned ts_inc;
pjmedia_rect_size vid_size;
const pjmedia_frame *frame;
unsigned frame_size;

pj_bool_t is_rendering;
void *render_buf;
unsigned render_buf_size;

gl_buffers *gl_buf;
GLView *gl_view;
Expand Down Expand Up @@ -229,9 +233,11 @@ - (void)render
}

pjmedia_vid_dev_opengl_draw(stream->gl_buf, stream->vid_size.w, stream->vid_size.h,
stream->frame->buf);
stream->render_buf);

[stream->ogl_context presentRenderbuffer:GL_RENDERBUFFER];

stream->is_rendering = PJ_FALSE;
}

- (void)finish_render
Expand Down Expand Up @@ -262,7 +268,9 @@ - (void)change_format
pjmedia_vid_dev_stream **p_vid_strm)
{
struct iosgl_stream *strm;
const pjmedia_video_format_info *vfi;
const pjmedia_video_format_detail *vfd;
pjmedia_video_apply_fmt_param vafp;
pj_status_t status = PJ_SUCCESS;
CGRect rect;

Expand Down Expand Up @@ -329,6 +337,17 @@ - (void)change_format
iosgl_stream_set_cap(&strm->base, PJMEDIA_VID_DEV_CAP_ORIENTATION,
&param->orient);
}

vfi = pjmedia_get_video_format_info(NULL, param->fmt.id);
if (!vfi) return PJMEDIA_EVID_BADFORMAT;
vafp.size = param->fmt.det.vid.size;
vafp.buffer = NULL;
if (vfi->apply_fmt(vfi, &vafp) != PJ_SUCCESS)
return PJMEDIA_EVID_BADFORMAT;

strm->frame_size = vafp.framebytes;
strm->render_buf_size = strm->frame_size;
strm->render_buf = pj_pool_alloc(strm->pool, strm->render_buf_size);

PJ_LOG(4, (THIS_FILE, "iOS OpenGL ES renderer successfully created"));

Expand Down Expand Up @@ -396,6 +415,7 @@ static pj_status_t iosgl_stream_set_cap(pjmedia_vid_dev_stream *s,

if (cap==PJMEDIA_VID_DEV_CAP_FORMAT) {
const pjmedia_video_format_info *vfi;
pjmedia_video_apply_fmt_param vafp;
pjmedia_format *fmt = (pjmedia_format *)pval;
iosgl_fmt_info *ifi;

Expand All @@ -406,8 +426,20 @@ static pj_status_t iosgl_stream_set_cap(pjmedia_vid_dev_stream *s,
fmt->id);
if (!vfi)
return PJMEDIA_EVID_BADFORMAT;

vafp.size = fmt->det.vid.size;
vafp.buffer = NULL;
if (vfi->apply_fmt(vfi, &vafp) != PJ_SUCCESS)
return PJMEDIA_EVID_BADFORMAT;

pjmedia_format_copy(&strm->param.fmt, fmt);

strm->frame_size = vafp.framebytes;
if (strm->render_buf_size < strm->frame_size) {
/* Realloc only when needed */
strm->render_buf_size = strm->frame_size;
strm->render_buf=pj_pool_alloc(strm->pool, strm->render_buf_size);
}

[strm->gl_view performSelectorOnMainThread:@selector(change_format)
withObject:nil waitUntilDone:YES];
Expand Down Expand Up @@ -480,10 +512,19 @@ static pj_status_t iosgl_stream_put_frame(pjmedia_vid_dev_stream *strm,
if (!stream->is_running)
return PJ_EINVALIDOP;

stream->frame = frame;
/* Prevent more than one async rendering task. */
if (stream->is_rendering)
return PJ_EIGNORED;

if (stream->frame_size >= frame->size)
pj_memcpy(stream->render_buf, frame->buf, frame->size);
else
pj_memcpy(stream->render_buf, frame->buf, stream->frame_size);

/* Perform OpenGL drawing in the main thread. */
stream->is_rendering = PJ_TRUE;
[stream->gl_view performSelectorOnMainThread:@selector(render)
withObject:nil waitUntilDone:YES];
withObject:nil waitUntilDone:NO];

return PJ_SUCCESS;
}
Expand Down Expand Up @@ -513,7 +554,7 @@ static pj_status_t iosgl_stream_destroy(pjmedia_vid_dev_stream *strm)

if (stream->is_running)
iosgl_stream_stop(strm);

if (stream->gl_view) {
[stream->gl_view performSelectorOnMainThread:@selector(deinit_gl)
withObject:nil waitUntilDone:YES];
Expand Down
1 change: 1 addition & 0 deletions pjsip-apps/src/pjsua/ios/ipjsua.xcodeproj/xcshareddata/xcschemes/ipjsua.xcscheme
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
ignoresPersistentStateOnLaunch = "NO"
debugDocumentVersioning = "YES"
debugServiceExtension = "internal"
enableGPUValidationMode = "1"
allowLocationSimulation = "YES">
<BuildableProductRunnable
runnableDebuggingMode = "0">
Expand Down

0 comments on commit 7231146

Please sign in to comment.