Fix issue #23 #24 #25 #26

nbs/04_baseline_models.ipynb

@@ -55,7 +55,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%nbdev_export\n",
+    "%nbdev_hide\n",
     "from fastcore import test\n",
     "import pandas as pd\n",
     "import numpy as np"

nbs/load.ipynb

@@ -450,7 +450,7 @@
     "        path = obj.default_storage_path/f'{hash_code}' if path is None else Path(path)/f'{hash_code}'\n",
     "        df.to_pickle(path)\n",
     "        obj.metadata['TS']['hash'] = hash_code\n",
-    "        obj.add_file(path)\n",
+    "        obj.add_file(str(path))\n",
     "        return obj"
    ]
   },
@@ -1167,6 +1167,18 @@
    "display_name": "Python 3",
    "language": "python",
    "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.9"
   }
  },
  "nbformat": 4,

nbs/utils.ipynb

@@ -40,7 +40,8 @@
     "import wandb\n",
     "import pickle\n",
     "import pandas as pd\n",
-    "import numpy as np"
+    "import numpy as np\n",
+    "import tensorflow as tf"
    ]
   },
   {
@@ -672,6 +673,85 @@
    "source": [
     "test_eq(bar.metadata['ref']['type'], str(type(foo)))"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Keras add plot_top_losses functionality"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "UsageError: Line magic function `%nbdev_export` not found.\n"
+     ]
+    }
+   ],
+   "source": [
+    "%nbdev_export\n",
+    "from timecluster_extension.visualization import *"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "UsageError: Line magic function `%nbdev_export` not found.\n"
+     ]
+    }
+   ],
+   "source": [
+    "%nbdev_export\n",
+    "@patch\n",
+    "def plot_top_losses(self:tf.keras.Sequential, validation_data, k, largest=True, return_fig=True, title_pos=0.99, **kwargs):\n",
+    "    \"Take the validation data of model self, compute the model losses for every item there, sort, and plot the results.\\\n",
+    "    If `largest` is True, the validation losses will be sorted from larger to lower. Once they are sorted, take the\\\n",
+    "    k first items based on this order and plot the predictions.\\\n",
+    "    If 'return_fig' is true, a Figure-set of plots is returned. If not, just showed on screen\"\n",
+    "    # Get a prediction with the validation_data\n",
+    "    pred_validation_data = self.predict(validation_data)\n",
+    "    # Calculate the MSE with respect to original_data\n",
+    "    mse_values = np.mean(np.square(validation_data - pred_validation_data), axis=(1,2))\n",
+    "    \n",
+    "    # Order the numpy array and take the top k.\n",
+    "    if largest:\n",
+    "        id_loss_values = mse_values.argsort()[-k:]\n",
+    "        txt_var = \"Largest MSE of the model for validation dataset\"\n",
+    "    else:\n",
+    "        id_loss_values = mse_values.argsort()[:k]\n",
+    "        txt_var = \"Smallest MSE of the model for validation dataset\"\n",
+    "        \n",
+    "    # Plot figures\n",
+    "    list_figs = [None] * k\n",
+    "    for i in range(k):\n",
+    "        title = txt_var + \" windoes_num: \" + str(id_loss_values[i])\n",
+    "        list_figs[i] = plot_validation_ts_ae(validation_data, \n",
+    "                                             pred_validation_data,\n",
+    "                                             title_str = title, \n",
+    "                                             title_pos = title_pos,\n",
+    "                                             window_num = id_loss_values[i],\n",
+    "                                             return_fig = True,\n",
+    "                                             **kwargs)\n",
+    "    \n",
+    "    # Returns\n",
+    "    if return_fig:\n",
+    "        return list_figs\n",
+    "    else:\n",
+    "        list_figs \n",
+    "        return None"
+   ]
   }
  ],
  "metadata": {

r_shiny_app/global.R

@@ -10,6 +10,7 @@ library(tidyr)
 library(dplyr)
 library(dygraphs)
 library(dbscan)
+library(shinyWidgets)
 
 # Python dependencies
 wandb = import("wandb")

r_shiny_app/server.R

@@ -120,6 +120,49 @@ shinyServer(function(input, output, session) {
         plt
     })
 
+    # Observe the events related to zoom the embedding graph:
+    ranges <- reactiveValues(x = NULL, y = NULL)
+    observeEvent(input$update_coord_graph,{
+        # Take input values
+        zoom_values <- list(xmin = input$x_min,
+                            xmax = input$x_max,
+                            ymin = input$y_min,
+                            ymax = input$y_max)
+        if (!is.null(zoom_values)) {
+            ranges$x <- c(zoom_values$xmin, zoom_values$xmax)
+            ranges$y <- c(zoom_values$ymin, zoom_values$ymax)
+
+        } else {
+            ranges$x <- NULL
+            ranges$y <- NULL
+        }
+    })
+
+    # Observe the events related to change the appearance of the embedding graph:
+    config_style <- reactiveValues(path_line_size = 0.08,
+                                   path_alpha = 5/10,
+                                   point_alpha = 1/10,
+                                   point_size = 1)
+
+    observeEvent(input$update_emb_graph,{
+        style_values <- list(path_line_size = input$path_line_size ,
+                             path_alpha = input$path_alpha,
+                             point_alpha = input$point_alpha,
+                             point_size = input$point_size )
+
+        if (!is.null(style_values)) {
+            config_style$path_line_size <- style_values$path_line_size
+            config_style$path_alpha <- style_values$path_alpha
+            config_style$point_alpha <- style_values$point_alpha
+            config_style$point_size <- style_values$point_size
+        } else {
+            config_style$path_line_size <- NULL
+            config_style$path_alpha <- NULL
+            config_style$point_alpha <- NULL
+            config_style$point_size <- NULL 
+        }
+    })
+
     # Reactive emb_plot
     emb_plot <- reactive({
         req(embeddings())
@@ -141,15 +184,24 @@ shinyServer(function(input, output, session) {
 
         plt <- ggplot(data = embs_) + 
             aes(x = xcoord, y = ycoord, fill = highlight, color = cluster) + 
-            geom_point(shape = 21) + 
+            geom_point(shape = 21,alpha = config_style$point_alpha, size = config_style$point_size) + 
             scale_shape(solid = FALSE) +
-            geom_path(size=0.08, colour = "#2F3B65") + 
+            geom_path(size=config_style$path_line_size, colour = "#2F3B65",alpha = config_style$path_alpha) + 
             guides() + 
             scale_fill_manual(values = c("TRUE" = "green", "FALSE" = "NA"))+
+            coord_cartesian(xlim = ranges$x, ylim = ranges$y, expand = TRUE)+
             theme(legend.position = "none",
                   panel.background = element_rect(fill = "white", colour = "black"))
 
         plt
+        # svgPanZoom(
+        #     svglite:::inlineSVG(
+        #         #will put on separate line but also need show
+        #         show(
+        #             plt
+        #         )
+        #     )
+        # )
     })
     ###
     # Outputs

r_shiny_app/ui.R

@@ -20,8 +20,38 @@ shinyUI(fluidPage(
             hr(),
             numericInput("minPts_hdbscan", label = "Define value of minPts in HDBSCAN",value =100),
             checkboxInput("show_clusters", label = "Calculate and show clusters", value = FALSE),
-            actionButton("update_clust", label = "Update clusters")
-
+            actionButton("update_clust", label = "Update clusters"),
+            hr(),
+            dropdownButton(
+                tags$b("Set new coordinates to zoom"),
+                splitLayout(cellWidths ="25%",
+                    knobInput(inputId = "x_min",label = "x_min",value = NA,min = -100,max = 100,
+                            width = "50",height = "50",displayPrevious = TRUE,lineCap = "round",
+                            fgColor = "#1d89ff",inputColor = "#428BCA"),
+                    knobInput(inputId = "x_max",label = "x_max",value = NA,min = -100,max = 100,
+                              width = "50",height = "50",displayPrevious = TRUE,lineCap = "round",
+                              fgColor = "#1d89ff",inputColor = "#428BCA"),
+                    knobInput(inputId = "y_min",label = "y_min",value = NA,min = -100,max = 100,
+                              width = "50",height = "50",displayPrevious = TRUE,lineCap = "round",
+                              fgColor = "#1d89ff",inputColor = "#428BCA"),
+                    knobInput(inputId = "y_max",label = "y_max",value = NA,min = -100,max = 100,
+                              width = "50",height = "50",displayPrevious = TRUE,lineCap = "round",
+                              fgColor = "#1d89ff",inputColor = "#428BCA")
+                    ),
+                actionBttn(inputId = "update_coord_graph",label = "Update coords.",style = "simple",
+                           color = "primary",icon = icon("binoculars"),size = "xs", block = TRUE),
+                hr(),
+                tags$b("Configure aestethics"),
+                sliderInput("path_line_size", label = "path_line_size", value = 0.08, min=0, max=5, step = 0.01),
+                sliderInput("path_alpha", label = "path_alpha", value = 5/10, min=0, max=1, step = 0.01),
+                sliderInput("point_alpha", label = "point_alpha", value = 1/10, min=0, max=1, step = 0.01),
+                sliderInput("point_size", label = "point_size", value = 1, min=0, max=10, step = 0.5),
+                actionBttn(inputId = "update_emb_graph",label = "Update aestethics",style = "simple",
+                           color = "primary",icon = icon("bar-chart"),size = "xs", block = TRUE),
+                circle = TRUE, status = "primary",
+                icon = icon("gear"), width = "300px",
+                tooltip = tooltipOptions(title = "Click to configure the appearance !")
+            )
         ),
         # Show a plot of the generated distribution
         mainPanel(

timecluster_extension/_nbdev.py

@@ -9,6 +9,8 @@
          "get_latent_features": "03_dimensionality_reduction.ipynb",
          "fget_UMAP_embeddings": "03_dimensionality_reduction.ipynb",
          "plot_embeddings": "03_dimensionality_reduction.ipynb",
+         "baseline_model_predictor": "04_baseline_models.ipynb",
+         "get_windows_mse": "04_baseline_models.ipynb",
          "fpreprocess_numeric_vars": "load.ipynb",
          "fread_and_concat": "load.ipynb",
          "fread_mining_monitoring_files": "load.ipynb",
@@ -22,6 +24,7 @@
          "remove_constant_columns": "utils.ipynb",
          "ReferenceArtifact": "utils.ipynb",
          "wandb.apis.public.Artifact.to_obj": "utils.ipynb",
+         "tf.keras.Sequential.plot_top_losses": "utils.ipynb",
          "plot_TS": "visualization.ipynb",
          "plot_validation_ts_ae": "visualization.ipynb"}
 

timecluster_extension/dr.py

@@ -66,10 +66,10 @@ def fget_UMAP_embeddings(input_data, **kwargs):
     return (embeddings, reducer)
 
 # Cell
-def plot_embeddings(embeddings, umap_params):
+def plot_embeddings(embeddings, umap_params, fig_size = (25,25)):
     "Plot 2D embeddings thorugh a connected scatter plot"
     df_embeddings = pd.DataFrame(embeddings, columns = ['x1', 'x2'])
-    fig = plt.figure(figsize=(10,10))
+    fig = plt.figure(figsize=(fig_size[0],fig_size[1]))
     ax = fig.add_subplot(111)
     ax.scatter(df_embeddings['x1'], df_embeddings['x2'], marker='o', facecolors='none', edgecolors='b', alpha=0.1)
     ax.plot(df_embeddings['x1'], df_embeddings['x2'], alpha=0.5, picker=1)

timecluster_extension/load.py

@@ -104,7 +104,7 @@ def from_df(cls, df, name, path=None, sd=None, ed=None, normalize=False, **kwarg
         path = obj.default_storage_path/f'{hash_code}' if path is None else Path(path)/f'{hash_code}'
         df.to_pickle(path)
         obj.metadata['TS']['hash'] = hash_code
-        obj.add_file(path)
+        obj.add_file(str(path))
         return obj
 
 # Cell