Task183 metrics refresh

.github/workflows/CI.yml

@@ -51,6 +51,7 @@ jobs:
           sudo apt install -y g++-11
           sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 90
           sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 90
+          sudo apt install libzip-dev
 
       - name: install-cpprest
         run: sudo apt install libcpprest-dev

.gitignore

@@ -1,6 +1,10 @@
 Build/
 build/
 
+# Large-files
+zip-examples
+model_data
+
 # IDE
 .vscode
 .idea
@@ -10,6 +14,7 @@ build/
 .DS_Store
 .AppleDouble
 .LSOverride
+*.entitlements
 
 # Javascript
 package-lock.json

client/scripts/requests.js

@@ -124,29 +124,60 @@ async function deleteConnection(sending_object) {
     return response
 }
 
-function trainRequest() {
-    if (!train_data) {
-        errorNotification("No training data was set.")
-    } else {
-        fetch(`http://${py_server_address}/train/${user_id}/${model_id}/0`, {
-            method: "PUT",
-            mode: "cors",
-            headers: {"Content-Type": "text/csv"},
-            body: train_data,
-        }).then(response => {
-            showBuildNotification(response.ok)
-            onTrainShowPredict(response.ok)
-            if (response.ok) {
-                setModelView("success")
-            } else {
-                setModelView("error")
-            }
+function uploadRequest() {
+    if (data_upload.files.length == 0) return
+    const file = data_upload.files[0]
+
+    fetch(`http://${py_server_address}/${user_id}/${model_id}`, {
+        method: "PATCH",
+        mode: "cors",
+        body: file,
+    }).then(response => {
+        if (!response.ok) {
+            Swal.fire({
+                position: "top-end",
+                icon: "error",
+                title: "Failed to upload data",
+                showConfirmButton: false,
+                timer: 1500,
+            })
+            console.error(`Failed to upload data for ${file.name}`)
+            return
+        }
+        Swal.fire({
+            position: "top-end",
+            icon: "success",
+            title: "Successfully uploaded",
+            showConfirmButton: false,
+            timer: 1500,
         })
-    }
+    })
+    setModelView("irrelevant")
+    // allow user to press a train button from now on
+    button_wrapper = document.getElementById("train-button")
+    button_wrapper.getElementsByTy
+    train_button = button_wrapper.children[0]
+    button_wrapper.removeAttribute("disabled")
+    train_button.removeAttribute("disabled")
+}
+
+function trainRequest() {
+    fetch(`http://${py_server_address}/train/${user_id}/${model_id}/0`, {
+        method: "PUT",
+        mode: "cors",
+    }).then(response => {
+        showBuildNotification(response.ok)
+        onTrainShowPredict(response.ok)
+        if (response.ok) {
+            setModelView("success")
+        } else {
+            setModelView("error")
+        }
+    })
 }
 
 async function predictRequest() {
-    if (csv_predict.files.length == 0) {
+    if (predict_button.files.length == 0) {
         errorNotification("Empty predict file.")
         return
     }
@@ -159,25 +190,34 @@ async function predictRequest() {
             showConfirmButton: true,
         })
     }
-    const file = csv_predict.files[0]
-    const text = await file.text()
-    const response = await fetch(
+    const file = predict_button.files[0]
+    let response = await fetch(
         `http://${py_server_address}/predict/${user_id}/${model_id}`,
         {
             method: "PUT",
             mode: "cors",
-            headers: {"Content-Type": "text/csv"},
-            body: text,
+            body: file,
         },
     )
+    if (!response.ok) {
+        Swal.fire("Error!", "Failed to upload the png image", "error")
+        console.error(
+            `Failed to upload the png with ${response.statusText}: ${responseJson.error}`,
+        )
+        return
+    }
+    response = await fetch(
+        `http://${py_server_address}/predict/${user_id}/${model_id}`,
+        {method: "GET", mode: "cors"},
+    )
     const responseJson = await response.json()
-    hideResult() // hide previous predict result
-    if (response.ok) onPredictShowResult(responseJson)
-    else {
-        Swal.fire("Error!", "Server is not responding now.", "error")
-        errorNotification("Failed to predict.\n" + responseJson.error)
-        console.log(
-            `Predict failed with ${response.statusText}: ${responseJson.error}`,
+    if (!response.ok) {
+        Swal.fire("Error!", "Failed to predict", "error")
+        console.error(
+            `Failed to predict with with ${response.statusText}: ${responseJson.error}`,
         )
+        return
     }
+    hideResult() // hide previous predict result
+    onPredictShowResult(responseJson)
 }

client/templates/main.html

@@ -113,7 +113,7 @@ <h2 onmousedown="return false">MLCraft</h2>
 
         <label class="button-wrapper">
           Upload
-          <input type="file" id="csv-upload-button" accept=".csv" />
+          <input type="file" id="dataset-upload-button" accept=".csv,.zip" />
         </label>
 
         <label class="button-wrapper" id="train-button" disabled>
@@ -123,7 +123,7 @@ <h2 onmousedown="return false">MLCraft</h2>
 
         <label class="button-wrapper" disabled>
           Predict
-          <input type="file" accept=".csv" id="csv-predict-button" disabled />
+          <input type="file" accept=".png" id="predict-button" disabled />
         </label>
         <div class="predict-result-wrapper" id="predict-result-wrapper">
           <output class="form-control-res" id="res-value-out" name="res-value" for="x-value y-value"></output>
@@ -329,6 +329,7 @@ <h3></h3>
           deleteLayer(layer_to_delete).then(response => {
             if (response.ok) {
               super.removeNodeId(id)
+              updateConnectionsInLayers(id.split("-")[1])
             } else {
               errorNotification("Failed to delete layer.\n Server is not responding now.")
             }
@@ -395,15 +396,15 @@ <h3></h3>
             target: "Target",
           }
           const parametersMap = {
-            linear: "bias=0;inFeatures=1;outFeatures=2",
+            linear: "bias=0;inFeatures=2;outFeatures=1",
             relu: "",
             pooling: "",
             layernorm: "axis=1",
             conv2d: "KernelSize=1;inChannels=1;outChannels=1",
-            data: "width=2",
-            output: "width=1",
-            loss: "width=1",
-            target: "width=1",
+            data: "shape=[2]",
+            output: "",
+            loss: "",
+            target: "shape=[1]",
           }
           const layer = {
             type: nameMap[name],
@@ -569,7 +570,7 @@ <h3></h3>
           if (response.ok) {
             editor.updateNodeDataFromId(connection.input_id, {
               DBID: data2.DBID,
-              Connections: [].concat(`${data1.DBID};${connection.output_id}`, data2.Connections), // "db_id;drawflow_id"
+              Connections: data2.Connections.concat(`${data1.DBID};${connection.output_id}`), // "db_id;drawflow_id"
               Parameters: data2.Parameters,
             })
           } else {
@@ -1011,6 +1012,8 @@ <h3></h3>
         metricsPage.appendChild(imageElement)
         URL.revokeObjectURL(imageUrl)
       }
+
+      setInterval(getMetricsFromServer, 10000)
     </script>
     <!--Метрики-->
 
@@ -1113,12 +1116,12 @@ <h3></h3>
 
     <!--JSON requests handling-->
     <script>
-      const csv_predict = document.getElementById("csv-predict-button")
-      csv_predict.onclick = () => {
-        csv_predict.value = null
+      const predict_button = document.getElementById("predict-button")
+      predict_button.onclick = () => {
+        predict_button.value = null
       }
 
-      csv_predict.addEventListener("input", predictRequest)
+      predict_button.addEventListener("input", predictRequest)
 
       function buildJsonFormData(form) {
         const jsonFormData = {}
@@ -1282,26 +1285,14 @@ <h3></h3>
       const showBuildNotification = build_status => {
         build_status ? buildSuccessful() : buildUnsuccessful()
       }
-      async function csvUploaded() {
-        if (csv_upload.files.length == 0) return
-        const file = csv_upload.files[0]
-        const text = await file.text()
-        // do whatever with the `text`
-        setModelView("irrelevant")
-        console.log(`Got goods: ${text}`)
-        train_data = text
-        // allow user to press a train button from now on
-        button_wrapper = document.getElementById("train-button")
-        button_wrapper.getElementsByTy
-        train_button = button_wrapper.children[0]
-        button_wrapper.removeAttribute("disabled")
-        train_button.removeAttribute("disabled")
+      const data_upload = document.getElementById("dataset-upload-button")
+      data_upload.onclick = () => {
+        data_upload.value = null
       }
-      const csv_upload = document.getElementById("csv-upload-button")
-      csv_upload.addEventListener("change", csvUploaded)
+      data_upload.addEventListener("change", uploadRequest)
 
       function onTrainShowPredict(training_successful) {
-        const predict_button = document.getElementById("csv-predict-button")
+        const predict_button = document.getElementById("predict-button")
         hideResult() // hide result when new training results arrive
         if (training_successful) {
           predict_button.removeAttribute("disabled")
@@ -1412,6 +1403,7 @@ <h3></h3>
 
         // Input selection
         writeIdOnConnections(input_selection_layer_id)
+
         let sortable_list = getSortableInputList(node)
         let child = sortable_list.lastElementChild
 
@@ -1440,8 +1432,9 @@ <h3></h3>
             new_input_order = []
             for (let i in sortable_list.childNodes) {
               try {
-                if (sortable_list.childNodes[i].textContent.includes("input")) {
-                  new_input_order.push(Number(sortable_list.childNodes[i].textContent.split(" ")[3]))
+                if (sortable_list.childNodes[i].textContent.length > 0) {
+                  console.log(sortable_list.childNodes[i].textContent)
+                  new_input_order.push(Number(sortable_list.childNodes[i].textContent))
                 }
               } catch (e) {
                 continue
@@ -1450,7 +1443,7 @@ <h3></h3>
             matchConnectionsAfterSelection(new_input_order)
           },
         })
-        document.getElementById("layer-id").innerHTML = "ID: " + layer_dbid.toString()
+
         document.getElementById("layer-data").style.display = "block"
         setupLayerParameters(node, correct_id)
       }
@@ -1651,6 +1644,25 @@ <h3></h3>
           }
         }
       }
+
+      function updateConnectionsInLayers(deleted_id) {
+        let nodes = document.getElementsByClassName("drawflow-node")
+        for (let i of nodes) {
+          const data1 = editor.getNodeFromId(i.id.split("-")[1]).data
+          let updatedConnections = []
+          for (let connection of data1.Connections) {
+            if (connection.split(";")[1] == deleted_id) {
+              continue
+            }
+            updatedConnections.unshift(connection)
+          }
+          editor.updateNodeDataFromId(i.id.split("-")[1], {
+            DBID: data1.DBID,
+            Connections: updatedConnections,
+            Parameters: data1.Parameters,
+          })
+        }
+      }
     </script>
 
     <!--Терминал-->

py_server/mlcraft/__init__.py

@@ -63,6 +63,7 @@ def make_app(config=None):
     app.register_error_handler(KeyError, key_error)  # no json field
     app.register_error_handler(ValueError, value_error)  # json wrong type
     app.register_error_handler(HTTPException, http_error)
+    app.register_error_handler(TimeoutError, timeout_error)
     app.register_error_handler(ConnectTimeout, timeout_error)
     app.register_error_handler(ConnectionError, connection_error)
 

py_server/mlcraft/check_dimensions.py

@@ -6,8 +6,8 @@
 
 
 class Data2dChecker:
-    def __init__(self, width):
-        self.output_shape = [-1, width]
+    def __init__(self, shape: list[int]):
+        self.output_shape = [-1] + shape
 
     def __call__(self):
         return True
@@ -60,7 +60,7 @@ def __call__(self, input_shape: list[int], targets_shape: list[int]):
 def create_checker(layer: dict):
     match layer["type"]:
         case "Data" | "Target":
-            return Data2dChecker(layer["parameters"]["width"])
+            return Data2dChecker(layer["parameters"]["shape"])
         case "Linear":
             return LinearChecker(
                 layer["parameters"]["inFeatures"], layer["parameters"]["outFeatures"]