From 200dfe46c3e40c34b3a38931bf82c20bfa18b370 Mon Sep 17 00:00:00 2001 From: jcorporation Date: Mon, 24 Jun 2024 20:43:29 +0200 Subject: [PATCH] Feat: Implement WebradioDB cache #1071 --- htdocs/js/apidoc.js | 10 +- src/compile_time.h.in | 2 +- src/lib/api.c | 1 + src/lib/webradio.c | 170 +++++++++++++++++++++++++++++++-- src/mpd_worker/api.c | 16 ++-- src/mpd_worker/webradiodb.c | 84 +++++++++------- src/mpd_worker/webradiodb.h | 2 +- src/mympd_api/timer_handlers.c | 1 + 8 files changed, 234 insertions(+), 52 deletions(-) diff --git a/htdocs/js/apidoc.js b/htdocs/js/apidoc.js index df7b4df71..c945efd8d 100644 --- a/htdocs/js/apidoc.js +++ b/htdocs/js/apidoc.js @@ -2275,7 +2275,13 @@ const APImethods = { } }, "MYMPD_API_WEBRADIODB_UPDATE": { - "desc": "Updates the full WebradioDB.", - "params": {} + "desc": "Updates the WebradioDB.", + "params": { + "force": { + "type": APItypes.bool, + "example": false, + "desc": "true = forces an update" + } + } } }; diff --git a/src/compile_time.h.in b/src/compile_time.h.in index 4ba659a91..caa105162 100644 --- a/src/compile_time.h.in +++ b/src/compile_time.h.in @@ -334,7 +334,7 @@ extern struct t_mympd_queue *mympd_api_queue; #define JSONRPC_UINT_MIN 0 #define JSONRPC_UINT_MAX UINT_MAX #define JSONRPC_STR_MAX 3000 -#define JSONRPC_KEY_MAX 50 +#define JSONRPC_KEY_MAX 500 #define JSONRPC_ARRAY_MAX 1000 #define JSONRPC_TIME_MIN 0 // Do 1. Jan 01:00:00 CET 1970 #define JSONRPC_TIME_MAX 253402297169 // Fr 31. Dez 23:59:29 CET 9999 diff --git a/src/lib/api.c b/src/lib/api.c index 87f28ce2f..8ae99b441 100644 --- a/src/lib/api.c +++ b/src/lib/api.c @@ -130,6 +130,7 @@ bool is_mympd_only_api_method(enum mympd_cmd_ids cmd_id) { case MYMPD_API_SETTINGS_GET: case MYMPD_API_CACHE_DISK_CLEAR: case MYMPD_API_CACHE_DISK_CROP: + case MYMPD_API_WEBRADIODB_UPDATE: return true; default: return false; diff --git a/src/lib/webradio.c b/src/lib/webradio.c index bee92fb79..0523ace7a 100644 --- a/src/lib/webradio.c +++ b/src/lib/webradio.c @@ -7,7 +7,11 @@ #include "compile_time.h" #include "src/lib/webradio.h" +#include "dist/mpack/mpack.h" +#include "src/lib/filehandler.h" +#include "src/lib/log.h" #include "src/lib/mem.h" +#include "src/lib/mpack.h" #include "src/lib/sds_extras.h" /** @@ -73,11 +77,91 @@ void webradio_free(rax *webradios) { * @return true on success, else false */ bool webradio_save_to_disk(struct t_config *config, rax *webradios, const char *filename) { - (void)config; - (void)webradios; - (void)filename; - //TODO: implement - return true; + if (webradios == NULL) { + MYMPD_LOG_DEBUG(NULL, "Webradio is NULL not saving anything"); + return true; + } + MYMPD_LOG_INFO(NULL, "Saving webradios to disc (%s)", filename); + mpack_writer_t writer; + sds tmp_file = sdscatfmt(sdsempty(), "%S/%s/%s.XXXXXX", config->workdir, DIR_WORK_TAGS, filename); + FILE *fp = open_tmp_file(tmp_file); + if (fp == NULL) { + FREE_SDS(tmp_file); + return false; + } + // init mpack + mpack_writer_init_stdfile(&writer, fp, true); + mpack_writer_set_error_handler(&writer, log_mpack_write_error); + mpack_start_array(&writer, (uint32_t)webradios->numele); + raxIterator iter; + raxStart(&iter, webradios); + raxSeek(&iter, "^", NULL, 0); + struct t_list_node *current; + while (raxNext(&iter)) { + mpack_build_map(&writer); + struct t_webradio_data *data = (struct t_webradio_data *)iter.data; + mpack_write_cstr(&writer, "Key"); + mpack_write_str(&writer, (char *)iter.key, (uint32_t)iter.key_len); + mpack_write_kv(&writer, "Name", data->name); + mpack_write_kv(&writer, "Image", data->image); + mpack_write_kv(&writer, "Homepage", data->homepage); + mpack_write_kv(&writer, "Country", data->country); + mpack_write_kv(&writer, "State", data->state); + mpack_write_kv(&writer, "Description", data->description); + mpack_write_cstr(&writer, "Genres"); + mpack_build_array(&writer); + current = data->genres.head; + while (current != NULL) { + mpack_write_cstr(&writer, current->key); + current = current->next; + } + mpack_complete_array(&writer); + mpack_write_cstr(&writer, "Languages"); + mpack_build_array(&writer); + current = data->languages.head; + while (current != NULL) { + mpack_write_cstr(&writer, current->key); + current = current->next; + } + mpack_complete_array(&writer); + mpack_write_cstr(&writer, "Streams"); + mpack_build_array(&writer); + current = data->uris.head; + while (current != NULL) { + mpack_build_map(&writer); + mpack_write_kv(&writer, "Uri", current->key); + mpack_write_kv(&writer, "Codec", current->value_p); + mpack_write_kv(&writer, "Bitrate", current->value_i); + mpack_complete_map(&writer); + current = current->next; + } + mpack_complete_array(&writer); + mpack_complete_map(&writer); + } + raxStop(&iter); + mpack_finish_array(&writer); + // finish writing + bool rc = mpack_writer_destroy(&writer) != mpack_ok + ? false + : true; + if (rc == false) { + rm_file(tmp_file); + MYMPD_LOG_ERROR("default", "An error occurred encoding the data"); + FREE_SDS(tmp_file); + return false; + } + // rename tmp file + sds filepath = sdscatlen(sdsempty(), tmp_file, sdslen(tmp_file) - 7); + errno = 0; + if (rename(tmp_file, filepath) == -1) { + MYMPD_LOG_ERROR(NULL, "Rename file from \"%s\" to \"%s\" failed", tmp_file, filepath); + MYMPD_LOG_ERRNO(NULL, errno); + rm_file(tmp_file); + rc = false; + } + FREE_SDS(filepath); + FREE_SDS(tmp_file); + return rc; } /** @@ -87,8 +171,76 @@ bool webradio_save_to_disk(struct t_config *config, rax *webradios, const char * * @return newly allocated rax with webradios */ rax *webradio_read_from_disk(struct t_config *config, const char *filename) { - (void)config; - (void)filename; - //TODO: implement - return NULL; + sds filepath = sdscatfmt(sdsempty(), "%S/%s/%s", config->workdir, DIR_WORK_TAGS, filename); + if (testfile_read(filepath) == false) { + FREE_SDS(filepath); + return false; + } + rax *webradios = raxNew(); + + mpack_tree_t tree; + mpack_tree_init_filename(&tree, filepath, 0); + mpack_tree_set_error_handler(&tree, log_mpack_node_error); + FREE_SDS(filepath); + mpack_tree_parse(&tree); + mpack_node_t root = mpack_tree_root(&tree); + size_t len = mpack_node_array_length(root); + sds key = sdsempty(); + sds uri = sdsempty(); + sds codec = sdsempty(); + for (size_t i = 0; i < len; i++) { + mpack_node_t entry = mpack_node_array_at(root, i); + struct t_webradio_data *data = webradio_data_new(); + key = mpackstr_sdscat(key, entry, "Key"); + data->name = mpackstr_sds(entry, "Name"); + data->image = mpackstr_sds(entry, "Image"); + data->homepage = mpackstr_sds(entry, "Homepage"); + data->country = mpackstr_sds(entry, "Country"); + data->state = mpackstr_sds(entry, "State"); + data->description = mpackstr_sds(entry, "Description"); + mpack_node_t genre_node = mpack_node_map_cstr(entry, "Genres"); + size_t genre_len = mpack_node_array_length(genre_node); + for (size_t j = 0; j < genre_len; j++) { + mpack_node_t array_node = mpack_node_array_at(genre_node, j); + list_push_len(&data->languages, mpack_node_str(array_node), mpack_node_data_len(array_node),0, NULL, 0, NULL); + } + mpack_node_t lang_node = mpack_node_map_cstr(entry, "Languages"); + size_t lang_len = mpack_node_array_length(lang_node); + for (size_t j = 0; j < lang_len; j++) { + mpack_node_t array_node = mpack_node_array_at(lang_node, j); + list_push_len(&data->languages, mpack_node_str(array_node), mpack_node_data_len(array_node),0, NULL, 0, NULL); + } + mpack_node_t streams_node = mpack_node_map_cstr(entry, "Streams"); + size_t streams_len = mpack_node_array_length(streams_node); + for (size_t j = 0; j < streams_len; j++) { + mpack_node_t array_node = mpack_node_array_at(streams_node, j); + uri = mpackstr_sdscat(uri, array_node, "Uri"); + codec = mpackstr_sdscat(codec, array_node, "Codec"); + int64_t bitrate = mpack_node_int(mpack_node_map_cstr(array_node, "Bitrate")); + list_push(&data->uris, uri, bitrate, codec, NULL); + sdsclear(uri); + sdsclear(codec); + } + if (raxTryInsert(webradios, (unsigned char *)key, strlen(key), data, NULL) == 0) { + // insert error + MYMPD_LOG_ERROR(NULL, "Duplicate WebradioDB key found: %s", key); + webradio_data_free(data); + } + sdsclear(key); + } + FREE_SDS(key); + FREE_SDS(uri); + FREE_SDS(codec); + // clean up and check for errors + bool rc = mpack_tree_destroy(&tree) != mpack_ok + ? false + : true; + if (rc == false) { + MYMPD_LOG_ERROR("default", "Reading webradios %s failed.", filename); + webradio_free(webradios); + return NULL; + } + + MYMPD_LOG_INFO(NULL, "Read %" PRIu64 " webradios %s from disc", webradios->numele, filename); + return webradios; } diff --git a/src/mpd_worker/api.c b/src/mpd_worker/api.c index 3d5b06442..cd6dce322 100644 --- a/src/mpd_worker/api.c +++ b/src/mpd_worker/api.c @@ -377,14 +377,16 @@ void mpd_worker_api(struct t_mpd_worker_state *mpd_worker_state) { } break; case MYMPD_API_WEBRADIODB_UPDATE: - response->data = jsonrpc_respond_message(response->data, request->cmd_id, request->id, - JSONRPC_FACILITY_PLAYLIST, JSONRPC_SEVERITY_INFO, "WebradioDB update started"); - push_response(response); - rc = mpd_worker_webradiodb_update(mpd_worker_state); - if (rc == false) { - send_jsonrpc_notify(JSONRPC_FACILITY_GENERAL, JSONRPC_SEVERITY_ERROR, MPD_PARTITION_ALL, "WebradioDB update failed"); + if (json_get_bool(request->data, "$.params.force", &bool_buf1, &parse_error) == true) { + response->data = jsonrpc_respond_message(response->data, request->cmd_id, request->id, + JSONRPC_FACILITY_PLAYLIST, JSONRPC_SEVERITY_INFO, "WebradioDB update started"); + push_response(response); + rc = mpd_worker_webradiodb_update(mpd_worker_state, bool_buf1); + if (rc == false) { + send_jsonrpc_notify(JSONRPC_FACILITY_GENERAL, JSONRPC_SEVERITY_ERROR, MPD_PARTITION_ALL, "WebradioDB update failed"); + } + async = true; } - async = true; break; default: response->data = jsonrpc_respond_message(response->data, request->cmd_id, request->id, diff --git a/src/mpd_worker/webradiodb.c b/src/mpd_worker/webradiodb.c index ff7fd76eb..8b9d63ae0 100644 --- a/src/mpd_worker/webradiodb.c +++ b/src/mpd_worker/webradiodb.c @@ -10,6 +10,7 @@ #include "dist/mjson/mjson.h" #include "dist/rax/rax.h" #include "src/lib/api.h" +#include "src/lib/filehandler.h" #include "src/lib/http_client.h" #include "src/lib/jsonrpc.h" #include "src/lib/log.h" @@ -19,7 +20,7 @@ // private definitions -static bool parse_webradiodb(sds str, rax *webradiodb); +static rax *parse_webradiodb(sds str); static bool icb_webradio_alternate(const char *path, sds key, sds value, int vtype, validate_callback vcb, void *userdata, struct t_jsonrpc_parse_error *error); static struct t_webradio_data *parse_webradiodb_data(sds str); @@ -27,11 +28,23 @@ static struct t_webradio_data *parse_webradiodb_data(sds str); // public functions /** - * Updates the webradioDB from the cloud + * Updates the WebradioDB from the cloud * @param mpd_worker_state mpd worker state + * @param force true = force update, false = update only if database is outdated * @return true on success, else false */ -bool mpd_worker_webradiodb_update(struct t_mpd_worker_state *mpd_worker_state) { +bool mpd_worker_webradiodb_update(struct t_mpd_worker_state *mpd_worker_state, bool force) { + if (force == false) { + sds filepath = sdscatfmt(sdsempty(), "%S/%s/%s", mpd_worker_state->config->workdir, DIR_WORK_TAGS, FILENAME_WEBRADIODB); + time_t mtime = get_mtime(filepath); + FREE_SDS(filepath); + time_t now = time(NULL); + if (mtime > now - 86400) { + MYMPD_LOG_INFO(NULL, "WebradioDB is already up-to-date."); + return true; + } + } + struct mg_client_request_t http_request = { .method = "GET", .uri = WEBRADIODB_URI, @@ -45,20 +58,18 @@ bool mpd_worker_webradiodb_update(struct t_mpd_worker_state *mpd_worker_state) { http_client_response_clear(&http_response); return false; } - rax *webradiodb = raxNew(); - bool rc = parse_webradiodb(http_response.body, webradiodb); + rax *webradiodb = parse_webradiodb(http_response.body); http_client_response_clear(&http_response); - if (rc == false) { - webradio_free(webradiodb); - } - else { - webradio_save_to_disk(mpd_worker_state->config, webradiodb, FILENAME_WEBRADIODB); - struct t_work_request *request = create_request(REQUEST_TYPE_DISCARD, 0, 0, INTERNAL_API_WEBRADIODB_CREATED, NULL, mpd_worker_state->partition_state->name); - request->data = jsonrpc_end(request->data); - request->extra = (void *) webradiodb; - mympd_queue_push(mympd_api_queue, request, 0); + if (webradiodb == NULL) { + return false; } - return rc; + + webradio_save_to_disk(mpd_worker_state->config, webradiodb, FILENAME_WEBRADIODB); + struct t_work_request *request = create_request(REQUEST_TYPE_DISCARD, 0, 0, INTERNAL_API_WEBRADIODB_CREATED, NULL, "default"); + request->data = jsonrpc_end(request->data); + request->extra = (void *) webradiodb; + mympd_queue_push(mympd_api_queue, request, 0); + return true; } // private functions @@ -69,7 +80,7 @@ bool mpd_worker_webradiodb_update(struct t_mpd_worker_state *mpd_worker_state) { * @param webradiodb rax tree to populate * @return true on success, else false */ -static bool parse_webradiodb(sds str, rax *webradiodb) { +static rax *parse_webradiodb(sds str) { int koff; int klen; int voff; @@ -78,22 +89,27 @@ static bool parse_webradiodb(sds str, rax *webradiodb) { int off; sds key = sdsempty(); sds data_str = sdsempty(); + rax *webradiodb = raxNew(); for (off = 0; (off = mjson_next(str, (int)sdslen(str), off, &koff, &klen, &voff, &vlen, &vtype)) != 0; ) { key = sdscatlen(key, str + koff, (size_t)klen); data_str = sdscatlen(data_str, str + voff, (size_t)vlen); struct t_webradio_data *data = parse_webradiodb_data(data_str); - if (raxTryInsert(webradiodb, (unsigned char *)key, sdslen(key), data, NULL) == 0) { - // insert error - webradio_data_free(data); + if (data != NULL) { + if (raxTryInsert(webradiodb, (unsigned char *)key, sdslen(key), data, NULL) == 0) { + // insert error + MYMPD_LOG_ERROR(NULL, "Duplicate WebradioDB key found: %s", key); + webradio_data_free(data); + } } sdsclear(key); sdsclear(data_str); } FREE_SDS(key); FREE_SDS(data_str); - return true; + MYMPD_LOG_INFO(NULL, "Added %" PRIu64 " webradios", webradiodb->numele); + return webradiodb; } /** @@ -138,27 +154,31 @@ static bool icb_webradio_alternate(const char *path, sds key, sds value, int vty */ static struct t_webradio_data *parse_webradiodb_data(sds str) { struct t_webradio_data *data = webradio_data_new(); + struct t_jsonrpc_parse_error parse_error; + jsonrpc_parse_error_init(&parse_error); sds uri = NULL; sds codec = NULL; uint bitrate; - if (json_get_string(str, "$.Name", 1, URI_LENGTH_MAX, &data->name, vcb_isname, NULL) == false || - json_get_string(str, "$.Image", 1, URI_LENGTH_MAX, &data->image, vcb_isname, NULL) == false || - json_get_string(str, "$.Homepage", 1, URI_LENGTH_MAX, &data->homepage, vcb_isuri, NULL) == false || - json_get_string(str, "$.Country", 1, URI_LENGTH_MAX, &data->country, vcb_isname, NULL) == false || - json_get_string(str, "$.State", 1, URI_LENGTH_MAX, &data->state, vcb_isname, NULL) == false || - json_get_string(str, "$.Description", 1, URI_LENGTH_MAX, &data->state, vcb_istext, NULL) == false || - json_get_array_string(str, "$.Genre", &data->genres, vcb_isname, 64, NULL) == false || - json_get_array_string(str, "$.Languages", &data->languages, vcb_isname, 64, NULL) == false || - json_get_string(str, "$.StreamUri", 1, URI_LENGTH_MAX, &uri, vcb_isuri, NULL) == false || - json_get_string(str, "$.Codec", 1, URI_LENGTH_MAX, &codec, vcb_isname, NULL) == false || - json_get_uint_max(str, "$.Bitrate", &bitrate, NULL) == false) + if (json_get_string(str, "$.Name", 1, URI_LENGTH_MAX, &data->name, vcb_isname, &parse_error) == false || + json_get_string(str, "$.Image", 1, URI_LENGTH_MAX, &data->image, vcb_isname, &parse_error) == false || + json_get_string(str, "$.Homepage", 0, URI_LENGTH_MAX, &data->homepage, vcb_isuri, &parse_error) == false || + json_get_string(str, "$.Country", 0, URI_LENGTH_MAX, &data->country, vcb_isname, &parse_error) == false || + json_get_string(str, "$.State", 0, URI_LENGTH_MAX, &data->state, vcb_isname, &parse_error) == false || + json_get_string(str, "$.Description", 0, URI_LENGTH_MAX, &data->description, vcb_istext, &parse_error) == false || + json_get_array_string(str, "$.Genre", &data->genres, vcb_isname, 64, &parse_error) == false || + json_get_array_string(str, "$.Languages", &data->languages, vcb_isname, 64, &parse_error) == false || + json_get_string(str, "$.StreamUri", 1, URI_LENGTH_MAX, &uri, vcb_isuri, &parse_error) == false || + json_get_string(str, "$.Codec", 1, URI_LENGTH_MAX, &codec, vcb_isname, &parse_error) == false || + json_get_uint_max(str, "$.Bitrate", &bitrate, &parse_error) == false) { webradio_data_free(data); + jsonrpc_parse_error_clear(&parse_error); return NULL; } list_push(&data->uris, uri, bitrate, codec, NULL); - json_iterate_object(str, "$.alternativeStreams", icb_webradio_alternate, data, NULL, NULL, 64, NULL); + json_iterate_object(str, "$.alternativeStreams", icb_webradio_alternate, data, NULL, NULL, 64, &parse_error); FREE_SDS(uri); FREE_SDS(codec); + jsonrpc_parse_error_clear(&parse_error); return data; } diff --git a/src/mpd_worker/webradiodb.h b/src/mpd_worker/webradiodb.h index 5883ecabd..df6c534d3 100644 --- a/src/mpd_worker/webradiodb.h +++ b/src/mpd_worker/webradiodb.h @@ -11,6 +11,6 @@ #include -bool mpd_worker_webradiodb_update(struct t_mpd_worker_state *mpd_worker_state); +bool mpd_worker_webradiodb_update(struct t_mpd_worker_state *mpd_worker_state, bool force); #endif diff --git a/src/mympd_api/timer_handlers.c b/src/mympd_api/timer_handlers.c index 3751f3662..13efc57ab 100644 --- a/src/mympd_api/timer_handlers.c +++ b/src/mympd_api/timer_handlers.c @@ -220,6 +220,7 @@ static void timer_handler_caches_create(void) { static void timer_handler_webradiodb_update(void) { MYMPD_LOG_INFO(NULL, "Start timer_handler_webradiodb_update"); struct t_work_request *request = create_request(REQUEST_TYPE_DISCARD, 0, 0, MYMPD_API_WEBRADIODB_UPDATE, NULL, MPD_PARTITION_DEFAULT); + request->data = sdscat(request->data, "\"force\":false}}"); //only update if webradiodb is older than one day request->data = jsonrpc_end(request->data); push_request(request, 0); }