Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ XAV_DIR = c_src/xav
PRIV_DIR = $(MIX_APP_PATH)/priv
XAV_DECODER_SO = $(PRIV_DIR)/libxavdecoder.so
XAV_READER_SO = $(PRIV_DIR)/libxavreader.so
XAV_VIDEO_CONVERTER_SO = $(PRIV_DIR)/libxavvideoconverter.so

# uncomment to compile with debug logs
# XAV_DEBUG_LOGS = -DXAV_DEBUG=1
Expand All @@ -17,11 +18,14 @@ DECODER_SOURCES = $(XAV_DIR)/xav_decoder.c $(XAV_DIR)/decoder.c $(XAV_DIR)/video
READER_HEADERS = $(XAV_DIR)/xav_reader.h $(XAV_DIR)/reader.h $(XAV_DIR)/video_converter.h $(XAV_DIR)/audio_converter.h $(XAV_DIR)/utils.h $(XAV_DIR)/channel_layout.h
READER_SOURCES = $(XAV_DIR)/xav_reader.c $(XAV_DIR)/reader.c $(XAV_DIR)/video_converter.c $(XAV_DIR)/audio_converter.c $(XAV_DIR)/utils.c

VIDEO_CONVERTER_HEADERS = $(XAV_DIR)/xav_video_converter.h $(XAV_DIR)/video_converter.h $(XAV_DIR)/utils.h
VIDEO_CONVERTER_SOURCES = $(XAV_DIR)/xav_video_converter.c $(XAV_DIR)/video_converter.c $(XAV_DIR)/utils.c

CFLAGS = $(XAV_DEBUG_LOGS) -fPIC -shared
IFLAGS = -I$(ERTS_INCLUDE_DIR) -I$(XAV_DIR)
LDFLAGS = -lavcodec -lswscale -lavutil -lavformat -lavdevice -lswresample

# Falgs for MacOS
# Flags for MacOS
ifeq ($(shell uname -s),Darwin)
ifeq ($(shell uname -m),arm64)
IFLAGS += $$(pkg-config --cflags-only-I libavcodec libswscale libavutil libavformat libavdevice libswresample)
Expand All @@ -38,7 +42,7 @@ ifneq (,$(wildcard /etc/fedora-release))
LFLAGS += $$(pkg-config --libs-only-L libavcodec libswscale libavutil libavformat libavdevice libswresample)
endif

all: $(XAV_DECODER_SO) $(XAV_READER_SO)
all: $(XAV_DECODER_SO) $(XAV_READER_SO) $(XAV_VIDEO_CONVERTER_SO)

$(XAV_DECODER_SO): Makefile $(DECODER_SOURCES) $(DECODER_HEADERS)
mkdir -p $(PRIV_DIR)
Expand All @@ -48,6 +52,10 @@ $(XAV_READER_SO): Makefile $(READER_SOURCES) $(READER_HEADERS)
mkdir -p $(PRIV_DIR)
$(CC) $(CFLAGS) $(IFLAGS) $(LFLAGS) $(READER_SOURCES) -o $(XAV_READER_SO) $(LDFLAGS)

$(XAV_VIDEO_CONVERTER_SO): Makefile $(VIDEO_CONVERTER_SOURCES) $(VIDEO_CONVERTER_HEADERS)
mkdir -p $(PRIV_DIR)
$(CC) $(CFLAGS) $(IFLAGS) $(LFLAGS) $(VIDEO_CONVERTER_SOURCES) -o $(XAV_VIDEO_CONVERTER_SO) $(LDFLAGS)

format:
clang-format -i $(XAV_DIR)/*

Expand Down
16 changes: 16 additions & 0 deletions c_src/xav/utils.c
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,22 @@ ERL_NIF_TERM xav_nif_raise(ErlNifEnv *env, char *msg) {
return enif_raise_exception(env, reason);
}

int xav_get_atom(ErlNifEnv *env, ERL_NIF_TERM atom, char **value) {
unsigned int atom_len;
if (!enif_get_atom_length(env, atom, &atom_len, ERL_NIF_LATIN1)) {
return 0;
}

char *format = (char *)XAV_ALLOC((atom_len * 1) * sizeof(char *));
if (!enif_get_atom(env, atom, format, atom_len + 1, ERL_NIF_LATIN1)) {
XAV_FREE(format);
return 0;
}

*value = format;
return 1;
}

ERL_NIF_TERM xav_nif_audio_frame_to_term(ErlNifEnv *env, uint8_t **out_data, int out_samples,
int out_size, enum AVSampleFormat out_format, int pts) {
ERL_NIF_TERM data_term;
Expand Down
1 change: 1 addition & 0 deletions c_src/xav/utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
ERL_NIF_TERM xav_nif_ok(ErlNifEnv *env, ERL_NIF_TERM data_term);
ERL_NIF_TERM xav_nif_error(ErlNifEnv *env, char *reason);
ERL_NIF_TERM xav_nif_raise(ErlNifEnv *env, char *msg);
int xav_get_atom(ErlNifEnv *env, ERL_NIF_TERM atom, char **value);
ERL_NIF_TERM xav_nif_video_frame_to_term(ErlNifEnv *env, AVFrame *frame);
ERL_NIF_TERM xav_nif_audio_frame_to_term(ErlNifEnv *env, uint8_t **out_data, int out_samples,
int out_size, enum AVSampleFormat out_format, int pts);
91 changes: 65 additions & 26 deletions c_src/xav/video_converter.c
Original file line number Diff line number Diff line change
@@ -1,43 +1,82 @@
#include "video_converter.h"
#include "utils.h"

int video_converter_convert(AVFrame *src_frame, AVFrame **dst_frame,
enum AVPixelFormat out_format) {
int ret;
static inline unsigned int video_converter_resolution_changed(struct VideoConverter *converter, AVFrame *frame) {
return converter->in_format != frame->format ||
converter->in_width != frame->width ||
converter->in_height != frame->height;
}

*dst_frame = av_frame_alloc();
if (!*dst_frame) {
return -1;
struct VideoConverter *video_converter_alloc() {
struct VideoConverter *converter =
(struct VideoConverter *)XAV_ALLOC(sizeof(struct VideoConverter));
if(converter) {
converter->sws_ctx = NULL;
converter->dst_frame = av_frame_alloc();
}
return converter;
}

int video_converter_init(struct VideoConverter *converter, int in_width, int in_height,
enum AVPixelFormat in_format, enum AVPixelFormat out_format) {
converter->in_width = in_width;
converter->in_height = in_height;
converter->in_format = in_format;
converter->out_format = out_format;

av_frame_unref(converter->dst_frame);

(*dst_frame)->width = src_frame->width;
(*dst_frame)->height = src_frame->height;
(*dst_frame)->format = out_format;
(*dst_frame)->pts = src_frame->pts;
converter->dst_frame->width = in_width;
converter->dst_frame->height = in_height;
converter->dst_frame->format = out_format;

ret = av_frame_get_buffer(*dst_frame, 0);
if (ret < 0) {
int ret = av_frame_get_buffer(converter->dst_frame, 0);
if (ret < 0)
return ret;

converter->sws_ctx = sws_getContext(in_width, in_height, in_format, in_width, in_height, out_format,
SWS_BILINEAR, NULL, NULL, NULL);

if (!converter->sws_ctx) {
XAV_LOG_DEBUG("Couldn't get sws context");
return -1;
}

struct SwsContext *sws_ctx =
sws_getContext(src_frame->width, src_frame->height, src_frame->format, src_frame->width,
src_frame->height, out_format, SWS_BILINEAR, NULL, NULL, NULL);
return 0;
}

if (ret < 0) {
return ret;
int video_converter_convert(struct VideoConverter *converter, AVFrame *src_frame) {
int ret;

if (video_converter_resolution_changed(converter, src_frame)) {
XAV_LOG_DEBUG("Frame resolution changed");
sws_freeContext(converter->sws_ctx);
ret = video_converter_init(converter, src_frame->width, src_frame->height,
src_frame->format, converter->out_format);
if (ret < 0) {
return ret;
}
}

converter->dst_frame->pts = src_frame->pts;

// is this (const uint8_t * const*) cast really correct?
ret = sws_scale(sws_ctx, (const uint8_t *const *)src_frame->data, src_frame->linesize, 0,
src_frame->height, (*dst_frame)->data, (*dst_frame)->linesize);
return sws_scale(converter->sws_ctx, (const uint8_t *const *)src_frame->data, src_frame->linesize, 0,
src_frame->height, converter->dst_frame->data, converter->dst_frame->linesize);
}

if (ret < 0) {
av_frame_free(dst_frame);
sws_freeContext(sws_ctx);
return ret;
}
void video_converter_free(struct VideoConverter **converter) {
struct VideoConverter* vc = *converter;
if (vc != NULL) {
if (vc->sws_ctx != NULL) {
sws_freeContext((*converter)->sws_ctx);
}

sws_freeContext(sws_ctx);
if (vc->dst_frame != NULL) {
av_frame_free(&(*converter)->dst_frame);
}

return ret;
XAV_FREE(vc);
*converter = NULL;
}
}
18 changes: 17 additions & 1 deletion c_src/xav/video_converter.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,20 @@
#include <libswscale/swscale.h>
#include <stdint.h>

int video_converter_convert(AVFrame *src_frame, AVFrame **dst_frame, enum AVPixelFormat out_format);
struct VideoConverter {
struct SwsContext *sws_ctx;
int in_width;
int in_height;
enum AVPixelFormat in_format;
enum AVPixelFormat out_format;
AVFrame *dst_frame;
};

struct VideoConverter *video_converter_alloc();

int video_converter_init(struct VideoConverter* converter, int in_width, int in_height,
enum AVPixelFormat in_format, enum AVPixelFormat out_format);

int video_converter_convert(struct VideoConverter *converter, AVFrame *src_frame);

void video_converter_free(struct VideoConverter **converter);
35 changes: 28 additions & 7 deletions c_src/xav/xav_decoder.c
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
#include "xav_decoder.h"
#include "audio_converter.h"
#include "video_converter.h"

ErlNifResourceType *xav_decoder_resource_type;

static int init_audio_converter(struct XavDecoder *xav_decoder);
static int init_video_converter(struct XavDecoder *xav_decoder, AVFrame *frame);

void free_frames(AVFrame **frames, int size) {
for (int i = 0; i < size; i++) {
Expand Down Expand Up @@ -86,6 +86,7 @@ ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
enif_alloc_resource(xav_decoder_resource_type, sizeof(struct XavDecoder));
xav_decoder->decoder = NULL;
xav_decoder->ac = NULL;
xav_decoder->vc = NULL;
xav_decoder->out_audio_fmt = out_audo_fmt;
xav_decoder->out_video_fmt = out_video_fmt;
xav_decoder->out_sample_rate = out_sample_rate;
Expand Down Expand Up @@ -119,15 +120,20 @@ ERL_NIF_TERM convert(ErlNifEnv *env, struct XavDecoder *xav_decoder, AVFrame *fr
return xav_nif_video_frame_to_term(env, frame);
}

AVFrame *dst_frame;
ret = video_converter_convert(frame, &dst_frame, xav_decoder->out_video_fmt);
if (ret <= 0) {
return xav_nif_raise(env, "failed_to_decode");
if (xav_decoder->vc == NULL) {
ret = init_video_converter(xav_decoder, frame);
if (ret < 0) {
return xav_nif_raise(env, "failed_to_init_converter");
}
}

ret = video_converter_convert(xav_decoder->vc, frame);
if (ret < 0) {
return xav_nif_raise(env, "failed_to_convert");
}

frame_term = xav_nif_video_frame_to_term(env, dst_frame);
frame_term = xav_nif_video_frame_to_term(env, xav_decoder->vc->dst_frame);

av_frame_free(&dst_frame);
} else if (xav_decoder->decoder->media_type == AVMEDIA_TYPE_AUDIO) {
XAV_LOG_DEBUG("Converting audio to desired out format");

Expand Down Expand Up @@ -286,6 +292,17 @@ static int init_audio_converter(struct XavDecoder *xav_decoder) {
xav_decoder->out_audio_fmt);
}

static int init_video_converter(struct XavDecoder *xav_decoder, AVFrame *frame) {
xav_decoder->vc = video_converter_alloc();
if (xav_decoder->vc == NULL) {
XAV_LOG_DEBUG("Couldn't allocate video converter");
return -1;
}

return video_converter_init(xav_decoder->vc, frame->width, frame->height,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if this fails, we need to free video_converter. I can see that I implemented audio the same way but I think that's a bug

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, maybe that's not a bug as in our case, decoder's destructor will do the thing but I think we should fix this anyway i.e. free converter and leave xav_decoder->vc untouched.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes the destructor will free the converter in case of error.

frame->format, xav_decoder->out_video_fmt);
}

void free_xav_decoder(ErlNifEnv *env, void *obj) {
XAV_LOG_DEBUG("Freeing XavDecoder object");
struct XavDecoder *xav_decoder = (struct XavDecoder *)obj;
Expand All @@ -296,6 +313,10 @@ void free_xav_decoder(ErlNifEnv *env, void *obj) {
if (xav_decoder->ac != NULL) {
audio_converter_free(&xav_decoder->ac);
}

if (xav_decoder->vc != NULL) {
video_converter_free(&xav_decoder->vc);
}
}

static ErlNifFunc xav_funcs[] = {{"new", 4, new},
Expand Down
2 changes: 2 additions & 0 deletions c_src/xav/xav_decoder.h
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
#include "audio_converter.h"
#include "video_converter.h"
#include "decoder.h"

#include <libavutil/pixfmt.h>

struct XavDecoder {
struct Decoder *decoder;
struct AudioConverter *ac;
struct VideoConverter *vc;
enum AVPixelFormat out_video_fmt;
enum AVSampleFormat out_audio_fmt;
int out_sample_rate;
Expand Down
35 changes: 28 additions & 7 deletions c_src/xav/xav_reader.c
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
#include "xav_reader.h"
#include "video_converter.h"

static int init_audio_converter(struct XavReader *xav_reader);
static int init_video_converter(struct XavReader *xav_reader, AVFrame *frame);

ErlNifResourceType *xav_reader_resource_type;

ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
Expand Down Expand Up @@ -56,6 +57,7 @@ ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
enif_alloc_resource(xav_reader_resource_type, sizeof(struct XavReader));
xav_reader->reader = NULL;
xav_reader->ac = NULL;
xav_reader->vc = NULL;
xav_reader->out_format = out_format;
xav_reader->out_sample_rate = out_sample_rate;
xav_reader->out_channels = out_channels;
Expand Down Expand Up @@ -111,7 +113,7 @@ ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
} else if (xav_reader->reader->media_type == AVMEDIA_TYPE_VIDEO) {
ERL_NIF_TERM in_format_term =
enif_make_atom(env, av_get_pix_fmt_name(xav_reader->reader->c->pix_fmt));
ERL_NIF_TERM out_format_term = enif_make_atom(env, "rgb");
ERL_NIF_TERM out_format_term = enif_make_atom(env, "rgb24");
ERL_NIF_TERM framerate_num_term = enif_make_int(env, xav_reader->reader->framerate.num);
ERL_NIF_TERM framerate_den_term = enif_make_int(env, xav_reader->reader->framerate.den);
ERL_NIF_TERM framerate_term = enif_make_tuple(env, 2, framerate_num_term, framerate_den_term);
Expand Down Expand Up @@ -147,15 +149,19 @@ ERL_NIF_TERM next_frame(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
if (xav_reader->reader->media_type == AVMEDIA_TYPE_VIDEO) {
XAV_LOG_DEBUG("Converting video to RGB");

AVFrame *dst_frame;
ret = video_converter_convert(xav_reader->reader->frame, &dst_frame, AV_PIX_FMT_RGB24);
if (xav_reader->vc == NULL) {
ret = init_video_converter(xav_reader, xav_reader->reader->frame);
if (ret < 0) {
return xav_nif_raise(env, "failed_to_init_converter");
}
}

ret = video_converter_convert(xav_reader->vc, xav_reader->reader->frame);
if (ret <= 0) {
return xav_nif_raise(env, "failed_to_read");
}

frame_term = xav_nif_video_frame_to_term(env, dst_frame);

av_frame_free(&dst_frame);
frame_term = xav_nif_video_frame_to_term(env, xav_reader->vc->dst_frame);
} else if (xav_reader->reader->media_type == AVMEDIA_TYPE_AUDIO) {
XAV_LOG_DEBUG("Converting audio to desired out format");

Expand Down Expand Up @@ -277,6 +283,17 @@ static int init_audio_converter(struct XavReader *xav_reader) {
out_sample_fmt);
}

static int init_video_converter(struct XavReader *xav_reader, AVFrame *frame) {
xav_reader->vc = video_converter_alloc();
if (xav_reader->vc == NULL) {
XAV_LOG_DEBUG("Couldn't allocate video converter");
return -1;
}

return video_converter_init(xav_reader->vc, frame->width, frame->height,
frame->format, AV_PIX_FMT_RGB24);
}

void free_xav_reader(ErlNifEnv *env, void *obj) {
XAV_LOG_DEBUG("Freeing XavReader object");
struct XavReader *xav_reader = (struct XavReader *)obj;
Expand All @@ -287,6 +304,10 @@ void free_xav_reader(ErlNifEnv *env, void *obj) {
if (xav_reader->ac != NULL) {
audio_converter_free(&xav_reader->ac);
}

if (xav_reader->vc != NULL) {
video_converter_free(&xav_reader->vc);
}
}

static ErlNifFunc xav_funcs[] = {{"new", 6, new},
Expand Down
2 changes: 2 additions & 0 deletions c_src/xav/xav_reader.h
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
#include "audio_converter.h"
#include "video_converter.h"
#include "reader.h"

struct XavReader {
struct Reader *reader;
struct AudioConverter *ac;
struct VideoConverter *vc;
char *out_format;
int out_sample_rate;
int out_channels;
Expand Down
Loading