metrics

franzbischoff · Jun 10, 2023 · e538759 · e538759
1 parent 5163be2
commit e538759
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 13 deletions.
diff --git a/_contrast_profile/meta/meta b/_contrast_profile/meta/meta
@@ -1,11 +1,14 @@
 name|type|data|command|depend|seed|path|time|size|bytes|format|repository|iteration|parent|children|seconds|warnings|error
-.Random.seed|object|ea5142439dcdec95|||||||||||||||
+.Random.seed|object|665340f16c8c2950|||||||||||||||
 "%|||%"|function|031bda8ec980931b|||||||||||||||
 "%||NA%"|function|ef771f7e0b2b61dc|||||||||||||||
 activity|function|514ba81c8efb42b6|||||||||||||||
 ampl|function|22bb8917cc3362d0|||||||||||||||
 analysis_split|stem|6d6280eeeeaeaa09|3c9d74acfb239c7d|7d9b918f93b2e8de|1188165946||t19502.5658555796s|c1c66b69f95c736a|19205966|rds|local|group||analysis_split_7829d712*analysis_split_96143f68|0.207||
-assessment_split|stem|5367848bd2900085|1b41fd63223669d1|7d9b918f93b2e8de|-264140407||t19499.436301588s|54691e044ebfb63d|4805832|rds|local|group||assessment_split_46147acb*assessment_split_ac706e39|0.112||
+assessment_split|stem|5367848bd2900085|1b41fd63223669d1|7d9b918f93b2e8de|-264140407||t19502.5658556953s|54691e044ebfb63d|4805832|rds|local|group||assessment_split_46147acb*assessment_split_ac706e39|0.112||
+best_shapelets|pattern|3e680525b367f463|7a7e14f3c0872aa3||-261715276||||104899|rds|local|list||best_shapelets_082d95cb*best_shapelets_bc5dd927|0.226||
+best_shapelets_082d95cb|branch|c714dc25c4354b50|7a7e14f3c0872aa3|c543aae5458c5cd8|1797514474||t19518.3732832551s|73a2b92a437700aa|52732|rds|local|list|best_shapelets||0.138||
+best_shapelets_bc5dd927|branch|141d89f97104d940|7a7e14f3c0872aa3|b1a9398999af4fb5|893000502||t19518.3733171675s|b75045c6c2af8132|52167|rds|local|list|best_shapelets||0.088||
 classify_topk|function|1c48f3523ef2e30a|||||||||||||||
 clean_pred|function|2f000150c7903a2e|||||||||||||||
 clean_splits_data|function|62dff47d87f498a6|||||||||||||||
@@ -16,8 +19,8 @@ compute_arcs|function|1a4a6dc48008b78a|||||||||||||||
 compute_companion_stats|function|76613610273412a9|||||||||||||||
 compute_filters|function|7329a063b58bdd91|||||||||||||||
 compute_floss|function|2b4de81bfe11e55d|||||||||||||||
-compute_metrics_topk|function|a6523b2f4a20d346|||||||||||||||
-compute_overall_metric|function|ac70b78164855ebc|||||||||||||||
+compute_metrics_topk|function|696f4c42ab1f1f9d|||||||||||||||
+compute_overall_metric|function|524a4c19544f6bfd|||||||||||||||
 compute_s_profile_with_stats|function|2d5c844d8e0e655f|||||||||||||||
 compute_score_regimes|function|d6926f3b87fe56cf|||||||||||||||
 compute_streaming_profile|function|358c55ef8991349d|||||||||||||||
@@ -443,14 +446,18 @@ score_regimes_precision|function|750133b7a87964d6|||||||||||||||
 sd_r|function|edc210211e6db09c|||||||||||||||
 skip_graphics|object|909013909b1f03d4|||||||||||||||
 sprintf_transformer|function|2199f7feeee38815|||||||||||||||
-test_classifiers|pattern|488484bd98408016|d662ee19e1279577||-998301253||||1081|rds|local|list||test_classifiers_32880978*test_classifiers_8b513d75|30.504||
+test_classifiers|pattern|75d57b445c141bbb|8845c27433287427||-998301253||||1356|rds|local|list||test_classifiers_af14e4a5*test_classifiers_49f44e39|22.753||
+test_classifiers_15a0292b|branch|a12935294bc26b52|798e1432bbee03b0|55e770b0922bbfb1|-276228060||t19518.3606179225s|0f05aa39aab40022|347|rds|local|list|test_classifiers||11.069|Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.|
 test_classifiers_17e2a881|branch|78c176639587196e|812db5e8ae31b365|87638c71853d35a7|-1714255281||t19499.9332399218s|8144ee769374523f|514|rds|local|list|test_classifiers||3.666||
-test_classifiers_32880978|branch|2e3ffd8b88e03bd3|d662ee19e1279577|bc01984426128ae9|1439274487||t19513.7056631841s|bd37230314bbd1ee|539|rds|local|list|test_classifiers||18.309||
-test_classifiers_8b513d75|branch|97da4bbc2926b624|d662ee19e1279577|3530e9f18c1df494|-1633378582||t19513.7058483706s|9264840afbbafcaf|542|rds|local|list|test_classifiers||12.195||
+test_classifiers_32880978|branch|0f0c89ede3b55b33|11a5c7057a48a0cd|05a3e892e7362247|1439274487||t19518.2718028674s|febc0ae0f71ad275|1093|rds|local|list|test_classifiers||37.525||
+test_classifiers_35489b4b|branch|948b2fe4c59e5de3|798e1432bbee03b0|f15a80049e856b56|1678955518||t19518.3607392328s|7e50109c4a0b197a|346|rds|local|list|test_classifiers||10.2|Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.. Unknown or uninitialised column data.|
+test_classifiers_49f44e39|branch|87d83215102d1a0c|8845c27433287427|e21efbd670e70917|-1318716107||t19518.3735870239s|543b38e1c4e5a97d|671|rds|local|list|test_classifiers||12.936||
+test_classifiers_8b513d75|branch|2132d4b8ea5bc91e|11a5c7057a48a0cd|24b73c53e1559987|-1633378582||t19518.2722385529s|af2be79594b747a3|846|rds|local|list|test_classifiers||30.419||
+test_classifiers_af14e4a5|branch|ff24ec37550a246d|8845c27433287427|4e8fa0472bec340f|-2140311902||t19518.373434298s|f42b1c92efa2e2c7|685|rds|local|list|test_classifiers||9.817||
 test_classifiers_c5478cfa|branch|782db239ed983387|812db5e8ae31b365|a17fab4abe701ba7|342121097||t19499.9333528825s|b694c8343349aca9|550|rds|local|list|test_classifiers||3.428||
-test_classifiers_self|pattern|51a59e91064ddb23|6cb959d12fb88a90||1817600917||||800953|rds|local|list||test_classifiers_self_39b962e9*test_classifiers_self_d6d065d7|24282.608||
+test_classifiers_self|pattern|0541fd55870aaf64|0657566555a6166f||1817600917||||523761|rds|local|list||test_classifiers_self_39b962e9*test_classifiers_self_d6d065d7|14401.843||
 test_classifiers_self_39b962e9|branch|728890029868069d|0657566555a6166f|700aba9d80ac1b68|1575096345||t19513.62794029s|8e456d6751ec086f|477454|rds|local|list|test_classifiers_self||24528.014||
-test_classifiers_self_d6d065d7|branch|a3fcec170cbf7cf1|6cb959d12fb88a90|d58427442c681063|-1110083193||t19511.7504811982s|efbbf8f07e0cb217|418264|rds|local|list|test_classifiers_self||12359.66||
+test_classifiers_self_d6d065d7|branch|54e2c34f1bf8c4c7|0657566555a6166f|5f63c0f7feb9a402|-1110083193||t19514.306941681s|5d813e5bb024b0c6|523761|rds|local|list|test_classifiers_self||14401.843||
 testing_split|stem|4612cf50bd1a8a2e|c15f1f750870070b|074e071513f40f07|184895865||t19499.4362932547s|ab6e8a3595c8c4b1|788730|rds|local|vector|||0.002||
 topk_distance_profiles|function|275ec9c8aa99e412|||||||||||||||
 training_split|stem|972a76417af4261e|74d47449a574137b|074e071513f40f07|708679313||t19502.5658550009s|9b06d40e5906c4d0|2399817|rds|local|vector|||0.004||

diff --git a/scripts/_contrast_profile.R b/scripts/_contrast_profile.R
@@ -419,15 +419,41 @@ list(
       # bb <- dplyr::bind_rows(bb, aa)
       # i <- i + 1
       # summary(bb)
+      # library(corrgram)
       # corrgram(bb,
       #   order = FALSE,
       #   lower.panel = panel.ellipse, upper.panel = panel.cor,
       #   text.panel = panel.txt
       # )
+      # GGally::ggpairs(bb, aes(alpha = 0.05), lower = list(continuous = "smooth"))
     },
     pattern = map(find_shapelets, analysis_split),
     iteration = "list"
   ),
+  tar_target(
+    best_shapelets,
+    {
+      # Here we test the solutions we chose on the assessment split
+      # The final `model` we need is the shapelet
+
+      res <- list()
+      for (i in seq_len(var_vfolds)) {
+        aa <- tibble::as_tibble(purrr::transpose(test_classifiers_self[[i]]))
+        aa <- dplyr::mutate_all(aa, as.numeric)
+        aa <- dplyr::bind_cols(find_shapelets[[i]], aa) |>
+          # dplyr::select(-data) |>
+          dplyr::mutate(precision = tp / (tp + fp), recall = tp / (tp + fn)) |>
+          dplyr::mutate(coverage = as.numeric(coverage), redundancy = as.numeric(redundancy)) |>
+          dplyr::arrange(desc(precision), desc(recall)) |>
+          dplyr::slice_head(n = 1)
+
+        res[[i]] <- aa
+      }
+      res
+    },
+    pattern = map(test_classifiers_self, find_shapelets),
+    iteration = "list"
+  ),
   tar_target(
     test_classifiers,
     {
@@ -436,16 +462,15 @@ list(
       class(assessment_split) <- c("manual_rset", "rset", class(assessment_split))
 
       res <- list()
+
       for (i in seq_len(var_vfolds)) {
         fold <- rsample::get_rsplit(assessment_split, i)
-        best_shapelets <- purrr::pluck(find_shapelets, i)[1, ]
-
-        res[[i]] <- compute_metrics_topk(fold, best_shapelets, var_future_workers, TRUE)
+        res[[i]] <- compute_metrics_topk(fold, best_shapelets[[i]], var_future_workers, TRUE)
       }
       overall <- compute_overall_metric(res)
       list(fold = res, overall = overall)
     },
-    pattern = map(assessment_split, find_shapelets),
+    pattern = map(best_shapelets, assessment_split),
     iteration = "list"
   )
   # tar_target(

diff --git a/scripts/classification/pan_contrast.R b/scripts/classification/pan_contrast.R
@@ -663,6 +663,9 @@ compute_metrics_topk <- function(fold, shapelets, n_jobs = 1, progress = FALSE)
           }
         }
 
+        precision <- tp / (tp + fp)
+        recall <- tp / (tp + fn)
+        specificity <- tn / (tn + fp)
         accuracy <- (tp + tn) / (tp + tn + fp + fn)
         f1 <- 2 * tp / (2 * tp + fp + fn)
         p4 <- (4 * tp * tn) / (4 * tp * tn + (tp + tn) * (fp + fn))
@@ -673,6 +676,8 @@ compute_metrics_topk <- function(fold, shapelets, n_jobs = 1, progress = FALSE)
 
         list(
           tp = tp, fp = fp, tn = tn, fn = fn,
+          precision = precision, recall = recall,
+          specificity = specificity,
           accuracy = accuracy, f1 = f1,
           p4 = p4, mcc = mcc, kappa = kappa
         )
@@ -702,13 +707,17 @@ compute_overall_metric <- function(all_folds) {
   f1_micro <- (tm + fm) / 2
   f1_macro <- (ff / length(all_folds))
   f1_weighted <- ((tp + fp) * tm + (fn + tn) * fm) / (tp + tn + fp + fn)
+  pre <- tp / (tp + fp)
+  rec <- tp / (tp + fn)
+  spec <- tn / (tn + fp)
   acc <- (tp + tn) / (tp + tn + fp + fn)
   p4 <- (4 * tp * tn) / (4 * tp * tn + (tp + tn) * (fp + fn))
   mcc <- (tp * tn - fp * fn) / sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))
   kappa <- 2 * (tp * tn - fp * fn) / ((tp + fn) * (fn + tn) + (tp + fp) * (fp + tn))
 
   return(list(
     tp = tp, fp = fp, tn = tn, fn = fn,
+    precision = pre, recall = rec, specificity = spec,
     accuracy = acc, f1_micro = f1_micro, f1_macro = f1_macro, f1_weighted = f1_weighted,
     p4 = p4, mcc = mcc, kappa = kappa
   ))