From 327ba068f6e8657346068b9738cf5717499c3cd8 Mon Sep 17 00:00:00 2001 From: Francisco Bischoff Date: Sun, 13 Aug 2023 20:39:48 +0100 Subject: [PATCH] Build site. --- docs/figure/report.Rmd/figholdout-1.svg | 1310 ----------------------- docs/figure/report.Rmd/vtachy-1.svg | 653 ----------- docs/report.html | 491 ++++++++- 3 files changed, 447 insertions(+), 2007 deletions(-) delete mode 100644 docs/figure/report.Rmd/figholdout-1.svg delete mode 100644 docs/figure/report.Rmd/vtachy-1.svg diff --git a/docs/figure/report.Rmd/figholdout-1.svg b/docs/figure/report.Rmd/figholdout-1.svg deleted file mode 100644 index 9853f36..0000000 --- a/docs/figure/report.Rmd/figholdout-1.svg +++ /dev/null @@ -1,1310 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/figure/report.Rmd/vtachy-1.svg b/docs/figure/report.Rmd/vtachy-1.svg deleted file mode 100644 index df12a25..0000000 --- a/docs/figure/report.Rmd/vtachy-1.svg +++ /dev/null @@ -1,653 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/report.html b/docs/report.html index 1f5efda..3fa591a 100644 --- a/docs/report.html +++ b/docs/report.html @@ -440,7 +440,7 @@

on Jun 06, 2023

@@ -450,7 +450,7 @@

on Jun 06, 2023

  • -Checks +Checks
  • @@ -465,9 +465,9 @@

    on Jun 06, 2023

    Checks: -5 +7 -2 +0

    Knit directory: @@ -490,21 +490,16 @@

    on Jun 06, 2023

    - - -R Markdown file: uncommitted changes + + +R Markdown file: up-to-date

    -
    +
    -

    The R Markdown file has unstaged changes. -To know which version of the R Markdown file created these -results, you’ll want to first commit it to the Git repo. If -you’re still working on the analysis, you can ignore this -warning. When you’re finished, you can run -wflow_publish to commit the R Markdown file and -build the HTML.

    +

    Great! Since the R Markdown file has been committed to the Git repository, you +know the exact version of the code that produced these results.

    @@ -561,25 +556,16 @@

    on Jun 06, 2023

    - - -Cache: detected + + +Cache: none

    -
    +
    -The following chunks had caches available: -
      -
    • -dist-data -
      • -
    -

    To ensure reproducibility of the results, delete the cache directory -report_cache -and re-run the analysis. To have workflowr automatically delete the cache -directory prior to building the file, set delete_cache = TRUE -when running wflow_build() or wflow_publish().

    +

    Nice! There were no cached chunks for this analysis, so you can be confident +that you successfully produced the results during this run.

    @@ -602,20 +588,20 @@

    on Jun 06, 2023

    - + -Repository version: 4437e7a +Repository version: 4887954

    -
    +

    Great! You are using Git for version control. Tracking code development and connecting the code version to the results is critical for reproducibility.

    -The results in this page were generated with repository version 4437e7a. +The results in this page were generated with repository version 4887954. See the Past versions tab to see a history of the changes made to the R Markdown and HTML files.

    @@ -675,7 +661,7 @@

    on Jun 06, 2023

    Ignored: _targets/meta/progress Ignored: _targets/objects/ Ignored: _targets/user/ - Ignored: analysis/report_cache/ + Ignored: analysis/figure/ Ignored: analysis/shiny/rsconnect/ Ignored: analysis/shiny_land/rsconnect/ Ignored: analysis/shiny_ventricular/rsconnect/ @@ -688,13 +674,8 @@

    on Jun 06, 2023

    Ignored: presentations/MEDCIDS21/MEDCIDS21_files/ Ignored: presentations/Report/Midterm-Report_cache/ Ignored: presentations/Report/Midterm-Report_files/ - Ignored: protocol/SecondReport_cache/ - Ignored: protocol/SecondReport_files/ Ignored: protocol/ThirdReport.tex - Ignored: protocol/ThirdReport_cache/ - Ignored: protocol/ThirdReport_files/ Ignored: protocol/_extensions/ - Ignored: protocol/_files/ Ignored: protocol/figure/ Ignored: renv/staging/ Ignored: src/RcppExports.cpp @@ -713,10 +694,6 @@

    on Jun 06, 2023

    Ignored: thesis/_bookdown_files/ Ignored: tmp/ -Unstaged changes: - Modified: analysis/regime_optimize.Rmd - Modified: analysis/report.Rmd -

    Note that any generated files, e.g. HTML, png, CSS, etc., are not included in @@ -765,6 +742,40 @@

    on Jun 06, 2023

    Rmd +4887954 + + +Francisco Bischoff + + +2023-08-13 + + +update report + + + + +html + + +4887954 + + +Francisco Bischoff + + +2023-08-13 + + +update report + + + + +Rmd + + 7bf2605 @@ -2495,6 +2506,17 @@

    3.4.3 Preparing the data

    +4887954 + + +Francisco Bischoff + + +2023-08-13 + + + + ba67135 @@ -3249,6 +3271,43 @@

    4.1.2 Interactions

    Figure 4.3: Variable interactions strength using feature importance ranking measure (FIRM) approach60. A) Shows strong interaction between regime_threshold and regime_landmark, mp_threshold and window_size, mp_threshold and regime_landmark. B) Refitting the model with these interactions taken into account, the strength is substantially reduced, except for the first, showing that indeed there is a strong correlation between those variables.

    +

    + +

    +
    +
    + + + + + + + + + + + + + + + +
    +Version + +Author + +Date +
    +4887954 + +Francisco Bischoff + +2023-08-13 +
    +
    +

    4.1.3 Importance

    @@ -3308,6 +3367,43 @@

    4.1.4 Importance analysis

    Figure 4.4: Variables importances using three different methods. A) Feature Importance Ranking Measure using ICE curves. B) Permutation method. C) SHAP (400 iterations). Line 1 refers to the original fit, and line 2 to the re-fit, taking into account the interactions between variables (fig-interaction?).

    +

    + +

    +
    +
    + + + + + + + + + + + + + + + +
    +Version + +Author + +Date +
    +4887954 + +Francisco Bischoff + +2023-08-13 +
    +
    +

    Fig. 4.5 and Fig. 4.6 show the effect of each feature on the FLOSS score. The more evident difference is the shape of the effect of time_constraint that initially suggested better results with larger values. However, removing the interactions seems to be a flat line.

    Based on Fig. 4.4 and Fig. 4.6 we can infer that:

      @@ -3324,12 +3420,86 @@

      4.1.4 Importance analysis

      Figure 4.5: This shows the effect each variable has on the FLOSS score. This plot doesn’t take into account the variable interactions.

    +

    + +

    +
    +
    + + + + + + + + + + + + + + + +
    +Version + +Author + +Date +
    +4887954 + +Francisco Bischoff + +2023-08-13 +
    +
    +
    This shows the effect each variable has on the FLOSS score, taking into account the interactions.

    Figure 4.6: This shows the effect each variable has on the FLOSS score, taking into account the interactions.

    +

    + +

    +
    +
    + + + + + + + + + + + + + + + +
    +Version + +Author + +Date +
    +4887954 + +Francisco Bischoff + +2023-08-13 +
    +
    +

    According to the FLOSS paper41, the window_size is indeed a feature that can be tuned; nevertheless, the results appear to be similar in a reasonably wide range of window sizes, up to a limit, consistent with our findings.

    @@ -3346,6 +3516,43 @@

    4.1.5.1 By recording

    Figure 4.7: Violin plot showing the distribution of the FLOSS score achieved by all tested models by recording. The left half shows the recordings that were difficult to predict (10% overall), whereas the right half shows the recordings that at least one model could achieve a good prediction (10% overall). The recordings are sorted (left-right) by the minimum (best) score achieved in descending order, and ties are sorted by the median of all recording scores. The blue color highlights recordings where models had an IQR variability of less than one. As a simple example, a recording with just one regime change, and the model predicts exactly one change, far from the truth, the score will be roughly 1.

    +

    + +

    +
    +
    + + + + + + + + + + + + + + + +
    +Version + +Author + +Date +
    +4887954 + +Francisco Bischoff + +2023-08-13 +
    +
    +

    Fig. 4.8 shows the best effort in predicting the most complex recordings. One information not declared before is that if the model does not predict any change, it will put a mark on the zero position. On the other side, the truth markers positioned at the beginning and the end of the recording were removed, as these locations lack information and do not represent a streaming setting.

    Prediction of the worst 10% of recordings (red is the truth, blue are the predictions). @@ -3353,6 +3560,43 @@

    4.1.5.1 By recording

    Figure 4.8: Prediction of the worst 10% of recordings (red is the truth, blue are the predictions).

    +

    + +

    +
    +
    + + + + + + + + + + + + + + + +
    +Version + +Author + +Date +
    +4887954 + +Francisco Bischoff + +2023-08-13 +
    +
    +

    Fig. 4.9 shows the best performances of the best recordings. Notice that there are recordings with a significant duration and few regime changes, making it hard for a “trivial model” to predict randomly.

    Prediction of the best 10% of recordings (red is the truth, blue are the predictions). @@ -3360,6 +3604,43 @@

    4.1.5.1 By recording

    Figure 4.9: Prediction of the best 10% of recordings (red is the truth, blue are the predictions).

    +

    + +

    +
    +
    + + + + + + + + + + + + + + + +
    +Version + +Author + +Date +
    +4887954 + +Francisco Bischoff + +2023-08-13 +
    +
    +

    4.1.5.2 By model

    @@ -3370,6 +3651,43 @@

    4.1.5.2 By model

    Figure 4.10: Violin plot showing the distribution of the FLOSS score achieved by all tested models during the inner ressample. The left half shows the models with the worst performances (10% overall), whereas the right half shows the models with the best performances (10% overall). The models are sorted (left-right) by the mean score (top) and by the median (below). Ties are sorted by the SD and IQR, respectively. The bluish colors highlights models with an SD below 3 and IQR below 1.

    +

    + +

    +
    +
    + + + + + + + + + + + + + + + +
    +Version + +Author + +Date +
    +4887954 + +Francisco Bischoff + +2023-08-13 +
    +
    +

    Fig. 4.11 the performance of the six best models. They are ordered from left to right, from the worst record to the best record. The top model is the one with the lowest mean across the scores. The blue line indicates the mean score, and the red line the median score. The scores above 3 are squished in the plot and colored according to the scale in the legend.

    Performances of the best 6 models across all inner resample of recordings. The recordings are ordered by score, from the worst to the best. Each plot shows one model, starting from the best one. The red line indicates the median score of the model. The blue line indicates the mean score of the model. The gray line limits the zero-score region. The plot is limited on the "y" axis, and the scores above this limit are shown in color. @@ -3377,6 +3695,43 @@

    4.1.5.2 By model

    Figure 4.11: Performances of the best 6 models across all inner resample of recordings. The recordings are ordered by score, from the worst to the best. Each plot shows one model, starting from the best one. The red line indicates the median score of the model. The blue line indicates the mean score of the model. The gray line limits the zero-score region. The plot is limited on the “y” axis, and the scores above this limit are shown in color.

    +

    + +

    +
    +
    + + + + + + + + + + + + + + + +
    +Version + +Author + +Date +
    +4887954 + +Francisco Bischoff + +2023-08-13 +
    +
    +

    4.1.5.3 The Holdout

    @@ -3556,6 +3911,43 @@

    4.2 Classification

    Figure 4.13: Set of shapelets for the classification task. Above shows the model number, the coverage (the proportion of true alarms that were detected) and redundancy during the analysis_split (inner resample), The Precision, the Specificity, and the Km during the testing_split (outer resample).

    +

    + +

    +
    +
    + + + + + + + + + + + + + + + +
    +Version + +Author + +Date +
    +4887954 + +Francisco Bischoff + +2023-08-13 +
    +
    +

    After the Inner Resampling is done, the best sets of shapelets are selected and evaluated on the Test Set without retraining a new Contrast Profile. Thus assessing the generalization of the shapelet set on new data.

    The criteria to select the best sets of shapelets was described on section 3.5.2 being the Precision the ranking criteria. It was also required that the set being present on more than one fold and in both repetitions. Also, the sets of shapelets that had a negative \(\kappa_m\) were discarded.

    The following results were obtained:

    @@ -4300,6 +4692,17 @@

    5.2 Regime change detection

    +4887954 + + +Francisco Bischoff + + +2023-08-13 + + + + 96dd528