diff --git a/applications/mp4box/fileimport.c b/applications/mp4box/fileimport.c index a4d5c2d0a6..689fe538c3 100644 --- a/applications/mp4box/fileimport.c +++ b/applications/mp4box/fileimport.c @@ -1846,7 +1846,7 @@ GF_Err import_file(GF_ISOFile *dest, char *inName, u32 import_flags, GF_Fraction e = gf_file_load_data(rvc_config, (u8 **) &data, &size); GOTO_EXIT("loading RVC config file") - gf_gz_compress_payload(&data, size, &size); + gf_gz_compress_payload_ex (&data, size, &size, 0, GF_FALSE, NULL, GF_TRUE); e |= gf_isom_set_rvc_config(dest, track, 1, 0, "application/rvc-config+xml+gz", data, size); gf_free(data); GOTO_EXIT("compressing and assigning RVC config") diff --git a/share/doc/man/gpac-filters.1 b/share/doc/man/gpac-filters.1 index 203324f0b6..d8808908fb 100644 --- a/share/doc/man/gpac-filters.1 +++ b/share/doc/man/gpac-filters.1 @@ -7048,7 +7048,11 @@ When enabled, a GET on a directory will return a simple HTML listing of the cont .br .br -Custom headers can be specified using .I hdrs, they apply to all requests. +Custom headers can be specified using .I hdrs, they apply to all requests. For more advanced control on requests, use a javascript binding (see .I js and howtos). +.br + +.br +Text files are compressed using gzip or deflate if the client accepts these encodings, unless .I no_z is set. .br .br @@ -7344,7 +7348,9 @@ ka (bool, default: true): keep input alive if failure in push mode .br hdrs (strl): additional HTTP headers to inject, even values are names, odd values are values .br -js (str): jaavscript logic for server +js (str): javascript logic for server +.br +zmax (uint, default: 50000): maximum uncompressed size allowed for gzip or deflate compression for text files (only enabled if client indicates it), 0 will disable compression .br .br @@ -8619,50 +8625,6 @@ f (strl): bitstream filters name - see filter help * (str): any possible options defined for AVBitstreamFilter and sub-classes. See gpac -hx ffbsf and gpac -hx ffbsf:* .br -.br -.SH ffbsf -.LP -.br -Description: FFMPEG BitStream filter -.br -Version: Lavu58.9.100 -.br - -.br -This filter provides bitstream filters (BSF) for compressed audio and video formats. -.br -See FFMPEG documentation (https://ffmpeg.org/documentation.html) for more details -.br -To list all supported bitstream filters for your GPAC build, use gpac -h ffbsf:*. -.br - -.br -Several BSF may be specified in .I f for different coding types. BSF not matching the coding type are silently ignored. -.br -When no BSF matches the input coding type, or when .I f is empty, the filter acts as a passthrough filter. -.br - -.br -Options are specified after the desired filters: -.br -- ffbsf:f=h264_metadata:video_full_range_flag=0 -.br -- ffbsf:f=h264_metadata,av1_metadata:video_full_range_flag=0:color_range=tv -.br - -.br -Note: Using BSFs on some media types (e.g. avc, hevc) may trigger creation of a reframer filter (e.g. rfnalu) -.br - -.br -.SH Options (expert): -.LP -.br -f (strl): bitstream filters name - see filter help -.br -* (str): any possible options defined for AVBitstreamFilter and sub-classes. See gpac -hx ffbsf and gpac -hx ffbsf:* -.br - .br .SH jsf .LP diff --git a/src/filters/in_rtp_stream.c b/src/filters/in_rtp_stream.c index 7e8359fd71..a5bef8c094 100644 --- a/src/filters/in_rtp_stream.c +++ b/src/filters/in_rtp_stream.c @@ -502,7 +502,7 @@ GF_RTPInStream *rtpin_stream_new(GF_RTPIn *rtp, GF_SDPMedia *media, GF_SDPInfo * gf_free(rvc_data); return NULL; #else - gf_gz_decompress_payload(rvc_data, rvc_size, &tmp->depacketizer->sl_map.rvc_config, &tmp->depacketizer->sl_map.rvc_config_size); + gf_gz_decompress_payload_ex(rvc_data, rvc_size, &tmp->depacketizer->sl_map.rvc_config, &tmp->depacketizer->sl_map.rvc_config_size, GF_TRUE); gf_free(rvc_data); #endif } else { diff --git a/src/filters/out_http.c b/src/filters/out_http.c index 501c65c430..348e83919a 100644 --- a/src/filters/out_http.c +++ b/src/filters/out_http.c @@ -108,7 +108,7 @@ typedef struct #endif GF_PropStringList rdirs; Bool close, hold, quit, post, dlist, ice, reopen, blockio; - u32 port, block_size, maxc, maxp, timeout, hmode, sutc, cors, max_client_errors, max_async_buf, ka; + u32 port, block_size, maxc, maxp, timeout, hmode, sutc, cors, max_client_errors, max_async_buf, ka, zmax; s32 max_cache_segs; GF_PropStringList hdrs; @@ -270,6 +270,8 @@ struct __httpout_session { HTTP_DIRInfo *dir_desc; + u8 *comp_data; + #ifdef GPAC_HAS_QJS JSValue obj; #endif @@ -469,6 +471,10 @@ static void httpout_close_session(GF_HTTPOutSession *sess, GF_Err code) gf_dm_sess_del(sess->http_sess); sess->http_sess = NULL; sess->socket = NULL; + if (sess->comp_data) { + gf_free(sess->comp_data); + sess->comp_data = NULL; + } sess->done = 1; sess->flush_close = 0; if (sess->cbk_close) @@ -1358,6 +1364,10 @@ static void httpout_sess_io(void *usr_cbk, GF_NETIO_Parameter *parameter) sess->nb_ranges = 0; sess->do_log = httpout_do_log(sess, parameter->reply); sess->dir_desc = NULL; + if (sess->comp_data) { + gf_free(sess->comp_data); + sess->comp_data=NULL; + } if (!sess->async_pending && sess->ctx->on_request) { sess->async_pending = 1; @@ -1916,6 +1926,14 @@ static void httpout_sess_io(void *usr_cbk, GF_NETIO_Parameter *parameter) sess->reply_code = 200; } + u32 content_encoding=0; + const char *encode = (char *) gf_dm_sess_get_header(sess->http_sess, "Accept-Encoding"); + if (encode) { + if (strstr(encode, "gzip")) content_encoding=1; + else if (strstr(encode, "deflate")) content_encoding=2; + } + + //copy range for logs before resetting session headers if (sess->do_log && range) { strncpy(szRange, range, 199); @@ -1981,6 +1999,28 @@ static void httpout_sess_io(void *usr_cbk, GF_NETIO_Parameter *parameter) sess->nb_ranges = 0; gf_dm_sess_set_header(sess->http_sess, "Cache-Control", "no-cache, no-store"); } + //setup compression if asked, only if no chunk transfer and no ranges + if (!is_head && !sess->use_chunk_transfer && !sess->nb_ranges + && !sess->file_in_progress && content_encoding + && (sess->file_size<=sess->ctx->zmax) + ) { + u8 *data; + u32 osize; + if (gf_file_load_data_filep(sess->resource, &data, &osize)==GF_OK) { + u32 comp_size=0; + //only compress text files + if (gf_utf8_is_legal(data, osize) + && (gf_gz_compress_payload_ex((u8 **) &data, osize, &comp_size, 0, GF_FALSE, NULL, (content_encoding==1) ? GF_TRUE : GF_FALSE) == GF_OK) + ) { + sess->comp_data = data; + data = NULL; + sess->file_size = comp_size; + sess->bytes_in_req = comp_size; + gf_dm_sess_set_header(sess->http_sess, "Content-Encoding", (content_encoding==1) ? "gzip" : "deflate"); + } + if (data) gf_free(data); + } + } //only put content length if not using chunk transfer - bytes_in_req may be > 0 if we have a byte range on a chunk-transfer session if (sess->bytes_in_req && !sess->use_chunk_transfer) { sprintf(szFmt, LLU, sess->bytes_in_req); @@ -3536,7 +3576,11 @@ static void httpout_process_session(GF_Filter *filter, GF_HTTPOutCtx *ctx, GF_HT if (to_read > (u64) sess->ctx->block_size) to_read = (u64) sess->ctx->block_size; - if (sess->resource) { + if (sess->comp_data) { + memcpy(sess->buffer, sess->comp_data+sess->file_pos, to_read); + read = to_read; + } + else if (sess->resource) { read = (u32) gf_fread(sess->buffer, (u32) to_read, sess->resource); //may happen when file writing is in progress if (!read) { @@ -3614,6 +3658,9 @@ static void httpout_process_session(GF_Filter *filter, GF_HTTPOutCtx *ctx, GF_HT if (sess->resource) gf_fclose(sess->resource); sess->resource = NULL; + if (sess->comp_data) gf_free(sess->comp_data); + sess->comp_data = NULL; + if (sess->nb_bytes) { GF_LOG(GF_LOG_INFO, GF_LOG_HTTP, ("[HTTPOut] Done sending %s to %s ("LLU"/"LLU" bytes)\n", sess->path, sess->peer_address, sess->nb_bytes, sess->bytes_in_req)); } @@ -4957,8 +5004,9 @@ static const GF_FilterArgs HTTPOutArgs[] = { OFFS(ka), "keep input alive if failure in push mode", GF_PROP_BOOL, "true", NULL, GF_FS_ARG_HINT_EXPERT}, { OFFS(hdrs), "additional HTTP headers to inject, even values are names, odd values are values ", GF_PROP_STRING_LIST, NULL, NULL, GF_FS_ARG_HINT_ADVANCED}, #ifdef GPAC_HAS_QJS - { OFFS(js), "jaavscript logic for server", GF_PROP_STRING, NULL, NULL, GF_FS_ARG_HINT_EXPERT}, + { OFFS(js), "javascript logic for server", GF_PROP_STRING, NULL, NULL, GF_FS_ARG_HINT_EXPERT}, #endif + { OFFS(zmax), "maximum uncompressed size allowed for gzip or deflate compression for text files (only enabled if client indicates it), 0 will disable compression", GF_PROP_UINT, "50000", NULL, GF_FS_ARG_HINT_EXPERT}, {0} }; @@ -4997,7 +5045,9 @@ GF_FilterRegister HTTPOutRegister = { "When disabled, a GET on a directory will fail.\n" "When enabled, a GET on a directory will return a simple HTML listing of the content inspired from Apache.\n" " \n" - "Custom headers can be specified using [-hdrs](), they apply to all requests.\n" + "Custom headers can be specified using [-hdrs](), they apply to all requests. For more advanced control on requests, use a javascript binding (see [-js]() and howtos).\n" + " \n" + "Text files are compressed using gzip or deflate if the client accepts these encodings, unless [-no_z]() is set.\n" " \n" "# Simple HTTP server\n" "In this mode, the filter does not need any input connection and exposes all files in the directories given by [-rdirs]().\n" diff --git a/src/filters/out_route.c b/src/filters/out_route.c index cfc8acc387..19e26332c1 100644 --- a/src/filters/out_route.c +++ b/src/filters/out_route.c @@ -1239,7 +1239,7 @@ static GF_Err routeout_check_service_updates(GF_ROUTEOutCtx *ctx, ROUTEService * if (serv->stsid_bundle) gf_free(serv->stsid_bundle); serv->stsid_bundle = (u8 *) payload_text; serv->stsid_bundle_size = 1 + (u32) strlen(payload_text); - gf_gz_compress_payload(&serv->stsid_bundle, serv->stsid_bundle_size, &serv->stsid_bundle_size); + gf_gz_compress_payload_ex(&serv->stsid_bundle, serv->stsid_bundle_size, &serv->stsid_bundle_size, 0, GF_FALSE, NULL, GF_TRUE); serv->stsid_bundle_toi = 0x80000000; //compressed if (manifest_updated) serv->stsid_bundle_toi |= (1<<18); @@ -1930,7 +1930,7 @@ static void routeout_send_lls(GF_ROUTEOutCtx *ctx) comp_size = 2*len; payload = gf_malloc(sizeof(char)*(comp_size+4)); pay_start = payload + 4; - gf_gz_compress_payload_ex((u8 **) &payload_text, len, &comp_size, 0, GF_FALSE, &pay_start, GF_FALSE); + gf_gz_compress_payload_ex((u8 **) &payload_text, len, &comp_size, 0, GF_FALSE, &pay_start, GF_TRUE); gf_free(payload_text); payload_text = NULL; @@ -2022,7 +2022,7 @@ static void routeout_send_lls(GF_ROUTEOutCtx *ctx) comp_size = 2*len; payload = gf_malloc(sizeof(char)*(comp_size+4)); pay_start = payload + 4; - gf_gz_compress_payload_ex((u8 **) &payload_text, len, &comp_size, 0, GF_FALSE, &pay_start, GF_FALSE); + gf_gz_compress_payload_ex((u8 **) &payload_text, len, &comp_size, 0, GF_FALSE, &pay_start, GF_TRUE); gf_free(payload_text); payload_text = NULL; diff --git a/src/isomedia/box_funcs.c b/src/isomedia/box_funcs.c index d7c18978ec..26993b9aa1 100644 --- a/src/isomedia/box_funcs.c +++ b/src/isomedia/box_funcs.c @@ -196,7 +196,7 @@ GF_Err gf_isom_box_parse_ex(GF_Box **outBox, GF_BitStream *bs, u32 parent_type, compressed_size = (u32) (size - 8 - extra_bytes); gf_bs_read_data(bs, compb, compressed_size); - e = gf_gz_decompress_payload_ex(compb, compressed_size, &uncomp_data, &osize, (do_uncompress==2) ? GF_TRUE : GF_FALSE); + e = gf_gz_decompress_payload_ex(compb, compressed_size, &uncomp_data, &osize, GF_FALSE); if (e) { gf_free(compb); GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] Failed to uncompress payload for box type %s (0x%08X)\n", gf_4cc_to_str(otype), otype)); diff --git a/src/isomedia/isom_store.c b/src/isomedia/isom_store.c index 0460e46178..db5090f8f8 100644 --- a/src/isomedia/isom_store.c +++ b/src/isomedia/isom_store.c @@ -385,7 +385,7 @@ GF_Err gf_isom_write_compressed_box(GF_ISOFile *mov, GF_Box *root_box, u32 repl_ *box_csize = (u32) root_box->size; gf_bs_get_content(comp_bs, &box_data, &box_size); - gf_gz_compress_payload_ex(&box_data, box_size, &comp_size, use_cmov ? 0 : 8, GF_FALSE, NULL, use_cmov); + gf_gz_compress_payload_ex(&box_data, box_size, &comp_size, use_cmov ? 0 : 8, GF_FALSE, NULL, GF_FALSE); if ((mov->compress_flags & GF_ISOM_COMP_FORCE_ALL) || (comp_size + COMP_BOX_COST_BYTES < box_size)) { if (bs) { if (use_cmov) { @@ -569,7 +569,7 @@ u64 GetMoovAndMetaSize(GF_ISOFile *movie, GF_List *writers) } gf_bs_get_content(bs, &box_data, &box_size); - gf_gz_compress_payload_ex(&box_data, box_size, &osize, 0, GF_FALSE, NULL, GF_TRUE); + gf_gz_compress_payload_ex(&box_data, box_size, &osize, 0, GF_FALSE, NULL, GF_FALSE); gf_free(box_data); gf_bs_del(bs); return osize + 8 + 8 + 12 + 12; diff --git a/src/isomedia/track.c b/src/isomedia/track.c index 4fbf56ff21..9688603dc7 100644 --- a/src/isomedia/track.c +++ b/src/isomedia/track.c @@ -211,7 +211,7 @@ GF_Err GetESD(GF_MovieBox *moov, GF_ISOTrackID trackID, u32 StreamDescIndex, GF_ esd->decoderConfig->rvc_config = (GF_DefaultDescriptor *) gf_odf_desc_new(GF_ODF_DSI_TAG); if (mime_type && !strcmp(mime_type, "application/rvc-config+xml+gz") ) { #if !defined(GPAC_DISABLE_ZLIB) - gf_gz_decompress_payload(rvc_cfg_data, rvc_cfg_size, &esd->decoderConfig->rvc_config->data, &esd->decoderConfig->rvc_config->dataLength); + gf_gz_decompress_payload_ex(rvc_cfg_data, rvc_cfg_size, &esd->decoderConfig->rvc_config->data, &esd->decoderConfig->rvc_config->dataLength, GF_TRUE); gf_free(rvc_cfg_data); #endif } else { diff --git a/src/media_tools/dsmcc.c b/src/media_tools/dsmcc.c index 4e81a02436..69ddda1e23 100644 --- a/src/media_tools/dsmcc.c +++ b/src/media_tools/dsmcc.c @@ -610,7 +610,7 @@ static GF_Err dsmcc_module_complete(GF_M2TS_DSMCC_OVERLORD* dsmcc_overlord,GF_M2 u32 uncomp_size; u8* uncompressed_data; - gf_gz_decompress_payload(dsmcc_module->buffer,dsmcc_module->byte_sift, &uncompressed_data, &uncomp_size); + gf_gz_decompress_payload_ex(dsmcc_module->buffer,dsmcc_module->byte_sift, &uncompressed_data, &uncomp_size, GF_TRUE); //dsmcc_process_biop_data(dsmcc_overlord,dsmcc_module,uncompressed_data,dsmcc_module->original_size); if(dsmcc_module->original_size == uncomp_size) { diff --git a/src/media_tools/route_dmx.c b/src/media_tools/route_dmx.c index 167bdd2d96..d8394623b0 100644 --- a/src/media_tools/route_dmx.c +++ b/src/media_tools/route_dmx.c @@ -1362,7 +1362,7 @@ static GF_Err gf_route_dmx_process_service_signaling(GF_ROUTEDmx *routedmx, GF_R } memcpy(routedmx->buffer, object->payload, object->total_length); raw_size = routedmx->unz_buffer_size; - e = gf_gz_decompress_payload(routedmx->buffer, object->total_length, &routedmx->unz_buffer, &raw_size); + e = gf_gz_decompress_payload_ex(routedmx->buffer, object->total_length, &routedmx->unz_buffer, &raw_size, GF_TRUE); if (e) { GF_LOG(GF_LOG_ERROR, GF_LOG_ROUTE, ("[ROUTE] Service %d failed to decompress signaling bundle: %s\n", s->service_id, gf_error_to_string(e) )); return e; @@ -1747,7 +1747,7 @@ static GF_Err gf_route_dmx_process_lls(GF_ROUTEDmx *routedmx) return GF_OK; } - e = gf_gz_decompress_payload(&routedmx->buffer[4], read-4, &routedmx->unz_buffer, &raw_size); + e = gf_gz_decompress_payload_ex(&routedmx->buffer[4], read-4, &routedmx->unz_buffer, &raw_size, GF_TRUE); if (e) { GF_LOG(GF_LOG_ERROR, GF_LOG_ROUTE, ("[ROUTE] Failed to decompress %s table: %s\n", name, gf_error_to_string(e) )); return e; diff --git a/src/scene_manager/scene_engine.c b/src/scene_manager/scene_engine.c index f0400e6562..f1148c7bd2 100644 --- a/src/scene_manager/scene_engine.c +++ b/src/scene_manager/scene_engine.c @@ -400,7 +400,7 @@ static GF_Err gf_seng_encode_dims_au(GF_SceneEngine *seng, u16 ESID, GF_List *co if (compress_dims) { #ifndef GPAC_DISABLE_ZLIB dims_header |= GF_DIMS_UNIT_C; - e = gf_gz_compress_payload(&buffer, buffer_len, &buffer_len); + e = gf_gz_compress_payload_ex(&buffer, buffer_len, &buffer_len, 0, GF_FALSE, NULL, GF_TRUE); GF_LOG(GF_LOG_DEBUG, GF_LOG_SCENE, ("/ compressed (%d)", buffer_len)); if (e) goto exit; #else diff --git a/src/utils/base_encoding.c b/src/utils/base_encoding.c index f09bed00be..3323c3f4b1 100644 --- a/src/utils/base_encoding.c +++ b/src/utils/base_encoding.c @@ -199,9 +199,9 @@ GF_Err gf_gz_compress_payload_ex(u8 **data, u32 data_len, u32 *max_size, u8 data stream.opaque = (voidpf)NULL; if (use_gz) { - err = deflateInit(&stream, 9); - } else { err = deflateInit2(&stream, 9, Z_DEFLATED, 16+MAX_WBITS, 8, Z_DEFAULT_STRATEGY); + } else { + err = deflateInit(&stream, 9); } if (err != Z_OK) { @@ -274,9 +274,9 @@ GF_Err gf_gz_decompress_payload_ex(u8 *data, u32 data_len, u8 **uncompressed_dat d_stream.avail_out = size; if (use_gz) { - err = inflateInit(&d_stream); - } else { err = inflateInit2(&d_stream, 16+MAX_WBITS); + } else { + err = inflateInit(&d_stream); } if (err == Z_OK) { diff --git a/testsuite b/testsuite index 9d2c6d24da..703c33ec19 160000 --- a/testsuite +++ b/testsuite @@ -1 +1 @@ -Subproject commit 9d2c6d24da480f4335c4bf301eb5ddc44dbaf87a +Subproject commit 703c33ec192e3981ca0bbf0d1c66f7afcc34315a