From 11975ab654da8062fac886cf589688008cd55b71 Mon Sep 17 00:00:00 2001 From: brindosch Date: Mon, 19 Aug 2019 15:50:11 +0200 Subject: [PATCH 1/6] Feat: Add image sender to webui This PR adds a new image sending feature to the webui and extends the api accordingly. In javascript the processing of images is horrible slow (without WASM), so the solution is at the API side with out-of-the-box power of Qt. - The image cmd accepts now a raw image that is encoded with base64. Supported are all image formats that qt supports (enough) - There is no real size limit. It will be automatically scaled down to max 2000px width or height according to aspect ratio - It's possible to scale down through a new "scale" property if developer wants that. - Test were successfull until 3MP pictues, 4k+ closes the websocket on browser side, so 2k is a decent value --- assets/webconfig/content/remote.html | 9 ++- assets/webconfig/i18n/de.json | 1 + assets/webconfig/i18n/en.json | 1 + assets/webconfig/js/content_remote.js | 25 +++++-- assets/webconfig/js/hyperion.js | 10 +-- assets/webconfig/js/ui_utils.js | 6 +- libsrc/api/JSONRPC_schema/schema-image.json | 14 +++- libsrc/api/JsonAPI.cpp | 80 ++++++++++++++++++--- libsrc/hyperion/PriorityMuxer.cpp | 2 +- 9 files changed, 121 insertions(+), 27 deletions(-) diff --git a/assets/webconfig/content/remote.html b/assets/webconfig/content/remote.html index 292fdf911..2ad894392 100644 --- a/assets/webconfig/content/remote.html +++ b/assets/webconfig/content/remote.html @@ -43,9 +43,12 @@

- - - + + + + + + diff --git a/assets/webconfig/i18n/de.json b/assets/webconfig/i18n/de.json index d3e882101..1408445e7 100644 --- a/assets/webconfig/i18n/de.json +++ b/assets/webconfig/i18n/de.json @@ -215,6 +215,7 @@ "remote_color_button_reset": "Farbe/Effekt zurücksetzen", "remote_color_label_color": "Farbe:", "remote_effects_label_effects": "Effekt:", + "remote_effects_label_picture" : "Bild:", "remote_adjustment_label": "Farbanpassung", "remote_adjustment_intro": "Verändere live Farbe/Helligkeit/Kompensation. $1", "remote_videoMode_label": "Video Modus", diff --git a/assets/webconfig/i18n/en.json b/assets/webconfig/i18n/en.json index aa96b4af5..f49d2a24d 100644 --- a/assets/webconfig/i18n/en.json +++ b/assets/webconfig/i18n/en.json @@ -214,6 +214,7 @@ "remote_color_button_reset" : "Reset Color/Effect", "remote_color_label_color" : "Color:", "remote_effects_label_effects" : "Effect:", + "remote_effects_label_picture" : "Picture:", "remote_adjustment_label" : "Color adjustment", "remote_adjustment_intro" : "Modifiy color/brightness/compensation during runtime. $1", "remote_videoMode_label" : "Video mode", diff --git a/assets/webconfig/js/content_remote.js b/assets/webconfig/js/content_remote.js index f019bd5b8..3eaf91658 100644 --- a/assets/webconfig/js/content_remote.js +++ b/assets/webconfig/js/content_remote.js @@ -6,6 +6,8 @@ $(document).ready(function() { var mappingList = window.serverSchema.properties.color.properties.imageToLedMappingType.enum; var duration = 0; var rgb = {r:255,g:0,b:0}; + var lastImgData = ""; + var lastFileName= ""; //create html createTable('ssthead', 'sstbody', 'sstcont'); @@ -115,7 +117,7 @@ $(document).ready(function() { if(priority > 254) continue; - if(priority < 254 && (compId == "EFFECT" || compId == "COLOR") ) + if(priority < 254 && (compId == "EFFECT" || compId == "COLOR" || compId == "IMAGE") ) clearAll = true; if (visible) @@ -139,6 +141,9 @@ $(document).ready(function() { case "COLOR": owner = $.i18n('remote_color_label_color')+' '+'
'; break; + case "IMAGE": + owner = $.i18n('remote_effects_label_picture')+' '+owner; + break; case "GRABBER": owner = $.i18n('general_comp_GRABBER')+': ('+owner+')'; break; @@ -161,7 +166,7 @@ $(document).ready(function() { var btn = ''; - if((compId == "EFFECT" || compId == "COLOR") && priority < 254) + if((compId == "EFFECT" || compId == "COLOR" || compId == "IMAGE") && priority < 254) btn += ''; if(btn_type != 'default') @@ -320,10 +325,20 @@ $(document).ready(function() { sendEffect(); }); + $("#remote_input_repimg").off().on("click", function(){ + if(lastImgData != "") + requestSetImage(lastImgData, duration, lastFileName); + }); + $("#remote_input_img").change(function(){ - readImg(this, function(src,width,height){ - console.log(src,width,height) - requestSetImage(src,width,height,duration) + readImg(this, function(src,fileName){ + lastFileName = fileName; + if(src.includes(",")) + lastImgData = src.split(",")[1]; + else + lastImgData = src; + + requestSetImage(lastImgData, duration, lastFileName); }); }); diff --git a/assets/webconfig/js/hyperion.js b/assets/webconfig/js/hyperion.js index fbccb33f7..fd708a8af 100644 --- a/assets/webconfig/js/hyperion.js +++ b/assets/webconfig/js/hyperion.js @@ -123,19 +123,19 @@ function initWebSocket() { var error = response.hasOwnProperty("error")? response.error : "unknown"; $(window.hyperion).trigger({type:"error",reason:error}); - console.log("[window.websocket::onmessage] "+error) + console.log("[window.websocket::onmessage] ",error) } } catch(exception_error) { $(window.hyperion).trigger({type:"error",reason:exception_error}); - console.log("[window.websocket::onmessage] "+exception_error) + console.log("[window.websocket::onmessage] ",exception_error) } }; window.websocket.onerror = function (error) { $(window.hyperion).trigger({type:"error",reason:error}); - console.log("[window.websocket::onerror] "+error) + console.log("[window.websocket::onerror] ",error) }; } } @@ -290,9 +290,9 @@ function requestSetColor(r,g,b,duration) sendToHyperion("color", "", '"color":['+r+','+g+','+b+'], "priority":'+window.webPrio+',"duration":'+validateDuration(duration)+',"origin":"'+window.webOrigin+'"'); } -function requestSetImage(data,width,height,duration) +function requestSetImage(data,duration,name) { - sendToHyperion("image", "", '"imagedata":"'+data+'", "imagewidth":'+width+',"imageheight":'+height+', "priority":'+window.webPrio+',"duration":'+validateDuration(duration)+''); + sendToHyperion("image", "", '"imagedata":"'+data+'", "priority":'+window.webPrio+',"duration":'+validateDuration(duration)+', "format":"auto", "origin":"'+window.webOrigin+'", "name":"'+name+'"'); } function requestSetComponentState(comp, state) diff --git a/assets/webconfig/js/ui_utils.js b/assets/webconfig/js/ui_utils.js index 651f4a5eb..6ba30cb9c 100644 --- a/assets/webconfig/js/ui_utils.js +++ b/assets/webconfig/js/ui_utils.js @@ -392,11 +392,11 @@ function readImg(input,cb) { if (input.files && input.files[0]) { var reader = new FileReader(); + // inject fileName property + reader.fileName = input.files[0].name reader.onload = function (e) { - var i = new Image(); - i.src = e.target.result; - cb(i.src,i.width,i.height); + cb(e.target.result, e.target.fileName); } reader.readAsDataURL(input.files[0]); } diff --git a/libsrc/api/JSONRPC_schema/schema-image.json b/libsrc/api/JSONRPC_schema/schema-image.json index 506747176..c42098f41 100644 --- a/libsrc/api/JSONRPC_schema/schema-image.json +++ b/libsrc/api/JSONRPC_schema/schema-image.json @@ -26,17 +26,27 @@ }, "imagewidth": { "type" : "integer", - "required": true, "minimum": 0 }, "imageheight": { "type" : "integer", - "required": true, "minimum": 0 }, "imagedata": { "type": "string", "required": true + }, + "format": { + "type": "string", + "enum" : ["auto"] + }, + "scale": { + "type": "integer", + "minimum" : 25, + "maximum" : 2000 + }, + "name": { + "type": "string" } }, "additionalProperties": false diff --git a/libsrc/api/JsonAPI.cpp b/libsrc/api/JsonAPI.cpp index 8efd5120f..a00ad78c9 100644 --- a/libsrc/api/JsonAPI.cpp +++ b/libsrc/api/JsonAPI.cpp @@ -112,7 +112,7 @@ bool JsonAPI::handleInstanceSwitch(const quint8& inst, const bool& forced) // // imageStream last state // if(_ledcolorsImageActive) // connect(_hyperion, &Hyperion::currentImage, this, &JsonAPI::setImage, Qt::UniqueConnection); -// +// // //ledColor stream last state // if(_ledcolorsLedsActive) // connect(_hyperion, &Hyperion::rawLedColors, this, &JsonAPI::streamLedcolorsUpdate, Qt::UniqueConnection); @@ -172,7 +172,7 @@ void JsonAPI::handleMessage(const QString& messageString, const QString& httpAut sendErrorReply("No Authorization", command, tan); return; } - + // switch over all possible commands and handle them if (command == "color") handleColorCommand (message, command, tan); else if (command == "image") handleImageCommand (message, command, tan); @@ -231,20 +231,84 @@ void JsonAPI::handleImageCommand(const QJsonObject& message, const QString& comm int duration = message["duration"].toInt(-1); int width = message["imagewidth"].toInt(); int height = message["imageheight"].toInt(); + int scale = message["scale"].toInt(-1); + QString format = message["format"].toString(); + QString imgName = message["name"].toString(""); + QString origin = message["origin"].toString("JsonRpc"); QByteArray data = QByteArray::fromBase64(QByteArray(message["imagedata"].toString().toUtf8())); - // check consistency of the size of the received data - if (data.size() != width*height*3) + // truncate name length + imgName.truncate(16); + + if(format == "auto") { - sendErrorReply("Size of image data does not match with the width and height", command, tan); - return; + QImage img = QImage::fromData(data); + if(img.isNull()) + { + sendErrorReply("Failed to parse picture, the file might be corrupted", command, tan); + return; + } + + // check for requested scale + if(scale > 24) + { + if(img.height() > scale) + { + img = img.scaledToHeight(scale); + } + if(img.width() > scale) + { + img = img.scaledToWidth(scale); + } + } + + // check if we need to force a scale + if(img.width() > 2000 || img.height() > 2000) + { + scale = 2000; + if(img.height() > scale) + { + img = img.scaledToHeight(scale); + } + if(img.width() > scale) + { + img = img.scaledToWidth(scale); + } + } + + width = img.width(); + height = img.height(); + + // extract image + img = img.convertToFormat(QImage::Format_ARGB32_Premultiplied); + data.clear(); + data.reserve(img.width() * img.height() * 3); + for (int i = 0; i < img.height(); ++i) + { + const QRgb * scanline = reinterpret_cast(img.scanLine(i)); + for (int j = 0; j < img.width(); ++j) + { + data.append((char) qRed(scanline[j])); + data.append((char) qGreen(scanline[j])); + data.append((char) qBlue(scanline[j])); + } + } + } + else + { + // check consistency of the size of the received data + if (data.size() != width*height*3) + { + sendErrorReply("Size of image data does not match with the width and height", command, tan); + return; + } } - // create ImageRgb + // copy image Image image(width, height); memcpy(image.memptr(), data.data(), data.size()); - _hyperion->registerInput(priority, hyperion::COMP_IMAGE, "JsonRpc@"+_peerAddress); + _hyperion->registerInput(priority, hyperion::COMP_IMAGE, origin+"@"+_peerAddress, imgName); _hyperion->setInputImage(priority, image, duration); // send reply diff --git a/libsrc/hyperion/PriorityMuxer.cpp b/libsrc/hyperion/PriorityMuxer.cpp index 1f174ed73..7982ea34d 100644 --- a/libsrc/hyperion/PriorityMuxer.cpp +++ b/libsrc/hyperion/PriorityMuxer.cpp @@ -299,7 +299,7 @@ void PriorityMuxer::setCurrentTime(void) newPriority = qMin(newPriority, infoIt->priority); // call timeTrigger when effect or color is running with timeout > 0, blacklist prio 255 - if(infoIt->priority < 254 && infoIt->timeoutTime_ms > 0 && (infoIt->componentId == hyperion::COMP_EFFECT || infoIt->componentId == hyperion::COMP_COLOR)) + if(infoIt->priority < 254 && infoIt->timeoutTime_ms > 0 && (infoIt->componentId == hyperion::COMP_EFFECT || infoIt->componentId == hyperion::COMP_COLOR || infoIt->componentId == hyperion::COMP_IMAGE)) emit signalTimeTrigger(); // as signal to prevent Threading issues ++infoIt; From 4a321a6fe9cdac35a33c1ae4b7ceaa42f0504d21 Mon Sep 17 00:00:00 2001 From: brindosch Date: Mon, 19 Aug 2019 16:03:46 +0200 Subject: [PATCH 2/6] PriorityMuxer should clearAll COMP_IMAGE too --- libsrc/hyperion/PriorityMuxer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsrc/hyperion/PriorityMuxer.cpp b/libsrc/hyperion/PriorityMuxer.cpp index 7982ea34d..b833e59d8 100644 --- a/libsrc/hyperion/PriorityMuxer.cpp +++ b/libsrc/hyperion/PriorityMuxer.cpp @@ -268,7 +268,7 @@ void PriorityMuxer::clearAll(bool forceClearAll) for(auto key : _activeInputs.keys()) { const InputInfo info = getInputInfo(key); - if ((info.componentId == hyperion::COMP_COLOR || info.componentId == hyperion::COMP_EFFECT) && key < PriorityMuxer::LOWEST_PRIORITY-1) + if ((info.componentId == hyperion::COMP_COLOR || info.componentId == hyperion::COMP_EFFECT || info.componentId == hyperion::COMP_IMAGE) && key < PriorityMuxer::LOWEST_PRIORITY-1) { clearInput(key); } From 1c79bb8f5740f0774d58934aa61bba8414652530 Mon Sep 17 00:00:00 2001 From: brindosch Date: Mon, 19 Aug 2019 20:33:15 +0200 Subject: [PATCH 3/6] Sync with master --- libsrc/api/JsonAPI.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libsrc/api/JsonAPI.cpp b/libsrc/api/JsonAPI.cpp index f4daba8b3..17e438593 100644 --- a/libsrc/api/JsonAPI.cpp +++ b/libsrc/api/JsonAPI.cpp @@ -235,7 +235,6 @@ void JsonAPI::handleImageCommand(const QJsonObject& message, const QString& comm int scale = message["scale"].toInt(-1); QString format = message["format"].toString(); QString imgName = message["name"].toString(""); - QString origin = message["origin"].toString("JsonRpc"); QByteArray data = QByteArray::fromBase64(QByteArray(message["imagedata"].toString().toUtf8())); // truncate name length @@ -309,7 +308,7 @@ void JsonAPI::handleImageCommand(const QJsonObject& message, const QString& comm Image image(width, height); memcpy(image.memptr(), data.data(), data.size()); - _hyperion->registerInput(priority, hyperion::COMP_IMAGE, origin); + _hyperion->registerInput(priority, hyperion::COMP_IMAGE, origin, imgName); _hyperion->setInputImage(priority, image, duration); // send reply From c1456a2fd549bfd9cb91a2753d2bdde814fd492b Mon Sep 17 00:00:00 2001 From: brindosch Date: Tue, 20 Aug 2019 14:08:29 +0200 Subject: [PATCH 4/6] Feat: Add browser capture capabilities --- assets/webconfig/index.html | 11 ++++ assets/webconfig/js/streamer.js | 102 ++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 assets/webconfig/js/streamer.js diff --git a/assets/webconfig/index.html b/assets/webconfig/index.html index 71b4c3d27..8d54b19b6 100644 --- a/assets/webconfig/index.html +++ b/assets/webconfig/index.html @@ -93,6 +93,16 @@

Hyperion Web Configuration requires Javascript. Please enable Javascript in