Skip to content

Commit b52115d

Browse files
gBillalgBillal
authored
Add video converter (#25)
1 parent a26f39a commit b52115d

17 files changed

+771
-43
lines changed

‎Makefile‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ XAV_DIR = c_src/xav
77
PRIV_DIR = $(MIX_APP_PATH)/priv
88
XAV_DECODER_SO = $(PRIV_DIR)/libxavdecoder.so
99
XAV_READER_SO = $(PRIV_DIR)/libxavreader.so
10+
XAV_VIDEO_CONVERTER_SO = $(PRIV_DIR)/libxavvideoconverter.so
1011

1112
# uncomment to compile with debug logs
1213
# XAV_DEBUG_LOGS = -DXAV_DEBUG=1
@@ -17,11 +18,14 @@ DECODER_SOURCES = $(XAV_DIR)/xav_decoder.c $(XAV_DIR)/decoder.c $(XAV_DIR)/video
1718
READER_HEADERS = $(XAV_DIR)/xav_reader.h $(XAV_DIR)/reader.h $(XAV_DIR)/video_converter.h $(XAV_DIR)/audio_converter.h $(XAV_DIR)/utils.h $(XAV_DIR)/channel_layout.h
1819
READER_SOURCES = $(XAV_DIR)/xav_reader.c $(XAV_DIR)/reader.c $(XAV_DIR)/video_converter.c $(XAV_DIR)/audio_converter.c $(XAV_DIR)/utils.c
1920

21+
VIDEO_CONVERTER_HEADERS = $(XAV_DIR)/xav_video_converter.h $(XAV_DIR)/video_converter.h $(XAV_DIR)/utils.h
22+
VIDEO_CONVERTER_SOURCES = $(XAV_DIR)/xav_video_converter.c $(XAV_DIR)/video_converter.c $(XAV_DIR)/utils.c
23+
2024
CFLAGS = $(XAV_DEBUG_LOGS) -fPIC -shared
2125
IFLAGS = -I$(ERTS_INCLUDE_DIR) -I$(XAV_DIR)
2226
LDFLAGS = -lavcodec -lswscale -lavutil -lavformat -lavdevice -lswresample
2327

24-
# Falgs for MacOS
28+
# Flags for MacOS
2529
ifeq ($(shell uname -s),Darwin)
2630
ifeq ($(shell uname -m),arm64)
2731
IFLAGS += $$(pkg-config --cflags-only-I libavcodec libswscale libavutil libavformat libavdevice libswresample)
@@ -38,7 +42,7 @@ ifneq (,$(wildcard /etc/fedora-release))
3842
LFLAGS += $$(pkg-config --libs-only-L libavcodec libswscale libavutil libavformat libavdevice libswresample)
3943
endif
4044

41-
all: $(XAV_DECODER_SO) $(XAV_READER_SO)
45+
all: $(XAV_DECODER_SO) $(XAV_READER_SO) $(XAV_VIDEO_CONVERTER_SO)
4246

4347
$(XAV_DECODER_SO): Makefile $(DECODER_SOURCES) $(DECODER_HEADERS)
4448
mkdir -p $(PRIV_DIR)
@@ -48,6 +52,10 @@ $(XAV_READER_SO): Makefile $(READER_SOURCES) $(READER_HEADERS)
4852
mkdir -p $(PRIV_DIR)
4953
$(CC) $(CFLAGS) $(IFLAGS) $(LFLAGS) $(READER_SOURCES) -o $(XAV_READER_SO) $(LDFLAGS)
5054

55+
$(XAV_VIDEO_CONVERTER_SO): Makefile $(VIDEO_CONVERTER_SOURCES) $(VIDEO_CONVERTER_HEADERS)
56+
mkdir -p $(PRIV_DIR)
57+
$(CC) $(CFLAGS) $(IFLAGS) $(LFLAGS) $(VIDEO_CONVERTER_SOURCES) -o $(XAV_VIDEO_CONVERTER_SO) $(LDFLAGS)
58+
5159
format:
5260
clang-format -i $(XAV_DIR)/*
5361

‎c_src/xav/utils.c‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,22 @@ ERL_NIF_TERM xav_nif_raise(ErlNifEnv *env, char *msg) {
2020
return enif_raise_exception(env, reason);
2121
}
2222

23+
int xav_get_atom(ErlNifEnv *env, ERL_NIF_TERM atom, char **value) {
24+
unsigned int atom_len;
25+
if (!enif_get_atom_length(env, atom, &atom_len, ERL_NIF_LATIN1)) {
26+
return 0;
27+
}
28+
29+
char *format = (char *)XAV_ALLOC((atom_len * 1) * sizeof(char *));
30+
if (!enif_get_atom(env, atom, format, atom_len + 1, ERL_NIF_LATIN1)) {
31+
XAV_FREE(format);
32+
return 0;
33+
}
34+
35+
*value = format;
36+
return 1;
37+
}
38+
2339
ERL_NIF_TERM xav_nif_audio_frame_to_term(ErlNifEnv *env, uint8_t **out_data, int out_samples,
2440
int out_size, enum AVSampleFormat out_format, int pts) {
2541
ERL_NIF_TERM data_term;

‎c_src/xav/utils.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
ERL_NIF_TERM xav_nif_ok(ErlNifEnv *env, ERL_NIF_TERM data_term);
1919
ERL_NIF_TERM xav_nif_error(ErlNifEnv *env, char *reason);
2020
ERL_NIF_TERM xav_nif_raise(ErlNifEnv *env, char *msg);
21+
int xav_get_atom(ErlNifEnv *env, ERL_NIF_TERM atom, char **value);
2122
ERL_NIF_TERM xav_nif_video_frame_to_term(ErlNifEnv *env, AVFrame *frame);
2223
ERL_NIF_TERM xav_nif_audio_frame_to_term(ErlNifEnv *env, uint8_t **out_data, int out_samples,
2324
int out_size, enum AVSampleFormat out_format, int pts);

‎c_src/xav/video_converter.c‎

Lines changed: 65 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,82 @@
11
#include "video_converter.h"
2+
#include "utils.h"
23

3-
int video_converter_convert(AVFrame *src_frame, AVFrame **dst_frame,
4-
enum AVPixelFormat out_format) {
5-
int ret;
4+
static inline unsigned int video_converter_resolution_changed(struct VideoConverter *converter, AVFrame *frame) {
5+
return converter->in_format != frame->format ||
6+
converter->in_width != frame->width ||
7+
converter->in_height != frame->height;
8+
}
69

7-
*dst_frame = av_frame_alloc();
8-
if (!*dst_frame) {
9-
return -1;
10+
struct VideoConverter *video_converter_alloc() {
11+
struct VideoConverter *converter =
12+
(struct VideoConverter *)XAV_ALLOC(sizeof(struct VideoConverter));
13+
if(converter) {
14+
converter->sws_ctx = NULL;
15+
converter->dst_frame = av_frame_alloc();
1016
}
17+
return converter;
18+
}
19+
20+
int video_converter_init(struct VideoConverter *converter, int in_width, int in_height,
21+
enum AVPixelFormat in_format, enum AVPixelFormat out_format) {
22+
converter->in_width = in_width;
23+
converter->in_height = in_height;
24+
converter->in_format = in_format;
25+
converter->out_format = out_format;
26+
27+
av_frame_unref(converter->dst_frame);
1128

12-
(*dst_frame)->width = src_frame->width;
13-
(*dst_frame)->height = src_frame->height;
14-
(*dst_frame)->format = out_format;
15-
(*dst_frame)->pts = src_frame->pts;
29+
converter->dst_frame->width = in_width;
30+
converter->dst_frame->height = in_height;
31+
converter->dst_frame->format = out_format;
1632

17-
ret = av_frame_get_buffer(*dst_frame, 0);
18-
if (ret < 0) {
33+
int ret = av_frame_get_buffer(converter->dst_frame, 0);
34+
if (ret < 0)
1935
return ret;
36+
37+
converter->sws_ctx = sws_getContext(in_width, in_height, in_format, in_width, in_height, out_format,
38+
SWS_BILINEAR, NULL, NULL, NULL);
39+
40+
if (!converter->sws_ctx) {
41+
XAV_LOG_DEBUG("Couldn't get sws context");
42+
return -1;
2043
}
2144

22-
struct SwsContext *sws_ctx =
23-
sws_getContext(src_frame->width, src_frame->height, src_frame->format, src_frame->width,
24-
src_frame->height, out_format, SWS_BILINEAR, NULL, NULL, NULL);
45+
return 0;
46+
}
2547

26-
if (ret < 0) {
27-
return ret;
48+
int video_converter_convert(struct VideoConverter *converter, AVFrame *src_frame) {
49+
int ret;
50+
51+
if (video_converter_resolution_changed(converter, src_frame)) {
52+
XAV_LOG_DEBUG("Frame resolution changed");
53+
sws_freeContext(converter->sws_ctx);
54+
ret = video_converter_init(converter, src_frame->width, src_frame->height,
55+
src_frame->format, converter->out_format);
56+
if (ret < 0) {
57+
return ret;
58+
}
2859
}
2960

61+
converter->dst_frame->pts = src_frame->pts;
62+
3063
// is this (const uint8_t * const*) cast really correct?
31-
ret = sws_scale(sws_ctx, (const uint8_t *const *)src_frame->data, src_frame->linesize, 0,
32-
src_frame->height, (*dst_frame)->data, (*dst_frame)->linesize);
64+
return sws_scale(converter->sws_ctx, (const uint8_t *const *)src_frame->data, src_frame->linesize, 0,
65+
src_frame->height, converter->dst_frame->data, converter->dst_frame->linesize);
66+
}
3367

34-
if (ret < 0) {
35-
av_frame_free(dst_frame);
36-
sws_freeContext(sws_ctx);
37-
return ret;
38-
}
68+
void video_converter_free(struct VideoConverter **converter) {
69+
struct VideoConverter* vc = *converter;
70+
if (vc != NULL) {
71+
if (vc->sws_ctx != NULL) {
72+
sws_freeContext((*converter)->sws_ctx);
73+
}
3974

40-
sws_freeContext(sws_ctx);
75+
if (vc->dst_frame != NULL) {
76+
av_frame_free(&(*converter)->dst_frame);
77+
}
4178

42-
return ret;
79+
XAV_FREE(vc);
80+
*converter = NULL;
81+
}
4382
}

‎c_src/xav/video_converter.h‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,20 @@
55
#include <libswscale/swscale.h>
66
#include <stdint.h>
77

8-
int video_converter_convert(AVFrame *src_frame, AVFrame **dst_frame, enum AVPixelFormat out_format);
8+
struct VideoConverter {
9+
struct SwsContext *sws_ctx;
10+
int in_width;
11+
int in_height;
12+
enum AVPixelFormat in_format;
13+
enum AVPixelFormat out_format;
14+
AVFrame *dst_frame;
15+
};
16+
17+
struct VideoConverter *video_converter_alloc();
18+
19+
int video_converter_init(struct VideoConverter* converter, int in_width, int in_height,
20+
enum AVPixelFormat in_format, enum AVPixelFormat out_format);
21+
22+
int video_converter_convert(struct VideoConverter *converter, AVFrame *src_frame);
23+
24+
void video_converter_free(struct VideoConverter **converter);

‎c_src/xav/xav_decoder.c‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
#include "xav_decoder.h"
22
#include "audio_converter.h"
3-
#include "video_converter.h"
43

54
ErlNifResourceType *xav_decoder_resource_type;
65

76
static int init_audio_converter(struct XavDecoder *xav_decoder);
7+
static int init_video_converter(struct XavDecoder *xav_decoder, AVFrame *frame);
88

99
void free_frames(AVFrame **frames, int size) {
1010
for (int i = 0; i < size; i++) {
@@ -86,6 +86,7 @@ ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
8686
enif_alloc_resource(xav_decoder_resource_type, sizeof(struct XavDecoder));
8787
xav_decoder->decoder = NULL;
8888
xav_decoder->ac = NULL;
89+
xav_decoder->vc = NULL;
8990
xav_decoder->out_audio_fmt = out_audo_fmt;
9091
xav_decoder->out_video_fmt = out_video_fmt;
9192
xav_decoder->out_sample_rate = out_sample_rate;
@@ -119,15 +120,20 @@ ERL_NIF_TERM convert(ErlNifEnv *env, struct XavDecoder *xav_decoder, AVFrame *fr
119120
return xav_nif_video_frame_to_term(env, frame);
120121
}
121122

122-
AVFrame *dst_frame;
123-
ret = video_converter_convert(frame, &dst_frame, xav_decoder->out_video_fmt);
124-
if (ret <= 0) {
125-
return xav_nif_raise(env, "failed_to_decode");
123+
if (xav_decoder->vc == NULL) {
124+
ret = init_video_converter(xav_decoder, frame);
125+
if (ret < 0) {
126+
return xav_nif_raise(env, "failed_to_init_converter");
127+
}
128+
}
129+
130+
ret = video_converter_convert(xav_decoder->vc, frame);
131+
if (ret < 0) {
132+
return xav_nif_raise(env, "failed_to_convert");
126133
}
127134

128-
frame_term = xav_nif_video_frame_to_term(env, dst_frame);
135+
frame_term = xav_nif_video_frame_to_term(env, xav_decoder->vc->dst_frame);
129136

130-
av_frame_free(&dst_frame);
131137
} else if (xav_decoder->decoder->media_type == AVMEDIA_TYPE_AUDIO) {
132138
XAV_LOG_DEBUG("Converting audio to desired out format");
133139

@@ -286,6 +292,17 @@ static int init_audio_converter(struct XavDecoder *xav_decoder) {
286292
xav_decoder->out_audio_fmt);
287293
}
288294

295+
static int init_video_converter(struct XavDecoder *xav_decoder, AVFrame *frame) {
296+
xav_decoder->vc = video_converter_alloc();
297+
if (xav_decoder->vc == NULL) {
298+
XAV_LOG_DEBUG("Couldn't allocate video converter");
299+
return -1;
300+
}
301+
302+
return video_converter_init(xav_decoder->vc, frame->width, frame->height,
303+
frame->format, xav_decoder->out_video_fmt);
304+
}
305+
289306
void free_xav_decoder(ErlNifEnv *env, void *obj) {
290307
XAV_LOG_DEBUG("Freeing XavDecoder object");
291308
struct XavDecoder *xav_decoder = (struct XavDecoder *)obj;
@@ -296,6 +313,10 @@ void free_xav_decoder(ErlNifEnv *env, void *obj) {
296313
if (xav_decoder->ac != NULL) {
297314
audio_converter_free(&xav_decoder->ac);
298315
}
316+
317+
if (xav_decoder->vc != NULL) {
318+
video_converter_free(&xav_decoder->vc);
319+
}
299320
}
300321

301322
static ErlNifFunc xav_funcs[] = {{"new", 4, new},

‎c_src/xav/xav_decoder.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
#include "audio_converter.h"
2+
#include "video_converter.h"
23
#include "decoder.h"
34

45
#include <libavutil/pixfmt.h>
56

67
struct XavDecoder {
78
struct Decoder *decoder;
89
struct AudioConverter *ac;
10+
struct VideoConverter *vc;
911
enum AVPixelFormat out_video_fmt;
1012
enum AVSampleFormat out_audio_fmt;
1113
int out_sample_rate;

‎c_src/xav/xav_reader.c‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
#include "xav_reader.h"
2-
#include "video_converter.h"
32

43
static int init_audio_converter(struct XavReader *xav_reader);
4+
static int init_video_converter(struct XavReader *xav_reader, AVFrame *frame);
5+
56
ErlNifResourceType *xav_reader_resource_type;
67

78
ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
@@ -56,6 +57,7 @@ ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
5657
enif_alloc_resource(xav_reader_resource_type, sizeof(struct XavReader));
5758
xav_reader->reader = NULL;
5859
xav_reader->ac = NULL;
60+
xav_reader->vc = NULL;
5961
xav_reader->out_format = out_format;
6062
xav_reader->out_sample_rate = out_sample_rate;
6163
xav_reader->out_channels = out_channels;
@@ -111,7 +113,7 @@ ERL_NIF_TERM new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
111113
} else if (xav_reader->reader->media_type == AVMEDIA_TYPE_VIDEO) {
112114
ERL_NIF_TERM in_format_term =
113115
enif_make_atom(env, av_get_pix_fmt_name(xav_reader->reader->c->pix_fmt));
114-
ERL_NIF_TERM out_format_term = enif_make_atom(env, "rgb");
116+
ERL_NIF_TERM out_format_term = enif_make_atom(env, "rgb24");
115117
ERL_NIF_TERM framerate_num_term = enif_make_int(env, xav_reader->reader->framerate.num);
116118
ERL_NIF_TERM framerate_den_term = enif_make_int(env, xav_reader->reader->framerate.den);
117119
ERL_NIF_TERM framerate_term = enif_make_tuple(env, 2, framerate_num_term, framerate_den_term);
@@ -147,15 +149,19 @@ ERL_NIF_TERM next_frame(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
147149
if (xav_reader->reader->media_type == AVMEDIA_TYPE_VIDEO) {
148150
XAV_LOG_DEBUG("Converting video to RGB");
149151

150-
AVFrame *dst_frame;
151-
ret = video_converter_convert(xav_reader->reader->frame, &dst_frame, AV_PIX_FMT_RGB24);
152+
if (xav_reader->vc == NULL) {
153+
ret = init_video_converter(xav_reader, xav_reader->reader->frame);
154+
if (ret < 0) {
155+
return xav_nif_raise(env, "failed_to_init_converter");
156+
}
157+
}
158+
159+
ret = video_converter_convert(xav_reader->vc, xav_reader->reader->frame);
152160
if (ret <= 0) {
153161
return xav_nif_raise(env, "failed_to_read");
154162
}
155163

156-
frame_term = xav_nif_video_frame_to_term(env, dst_frame);
157-
158-
av_frame_free(&dst_frame);
164+
frame_term = xav_nif_video_frame_to_term(env, xav_reader->vc->dst_frame);
159165
} else if (xav_reader->reader->media_type == AVMEDIA_TYPE_AUDIO) {
160166
XAV_LOG_DEBUG("Converting audio to desired out format");
161167

@@ -277,6 +283,17 @@ static int init_audio_converter(struct XavReader *xav_reader) {
277283
out_sample_fmt);
278284
}
279285

286+
static int init_video_converter(struct XavReader *xav_reader, AVFrame *frame) {
287+
xav_reader->vc = video_converter_alloc();
288+
if (xav_reader->vc == NULL) {
289+
XAV_LOG_DEBUG("Couldn't allocate video converter");
290+
return -1;
291+
}
292+
293+
return video_converter_init(xav_reader->vc, frame->width, frame->height,
294+
frame->format, AV_PIX_FMT_RGB24);
295+
}
296+
280297
void free_xav_reader(ErlNifEnv *env, void *obj) {
281298
XAV_LOG_DEBUG("Freeing XavReader object");
282299
struct XavReader *xav_reader = (struct XavReader *)obj;
@@ -287,6 +304,10 @@ void free_xav_reader(ErlNifEnv *env, void *obj) {
287304
if (xav_reader->ac != NULL) {
288305
audio_converter_free(&xav_reader->ac);
289306
}
307+
308+
if (xav_reader->vc != NULL) {
309+
video_converter_free(&xav_reader->vc);
310+
}
290311
}
291312

292313
static ErlNifFunc xav_funcs[] = {{"new", 6, new},

‎c_src/xav/xav_reader.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
#include "audio_converter.h"
2+
#include "video_converter.h"
23
#include "reader.h"
34

45
struct XavReader {
56
struct Reader *reader;
67
struct AudioConverter *ac;
8+
struct VideoConverter *vc;
79
char *out_format;
810
int out_sample_rate;
911
int out_channels;

0 commit comments

Comments
 (0)