From a4e49dd5b1bb847024f17fca4e92dd870814416d Mon Sep 17 00:00:00 2001 From: Markus Loecher Date: Sun, 18 Oct 2020 08:00:02 +0000 Subject: [PATCH] version 1.0.1 --- DESCRIPTION | 10 +- MD5 | 11 +- build/vignette.rds | Bin 216 -> 231 bytes inst/CITATION | 18 + inst/doc/rfVarImpOOB-vignette.Rmd | 2 +- inst/doc/rfVarImpOOB-vignette.html | 932 +++++++++++++++++++++++++++-- vignettes/rfVarImpOOB-vignette.Rmd | 2 +- 7 files changed, 915 insertions(+), 60 deletions(-) create mode 100644 inst/CITATION diff --git a/DESCRIPTION b/DESCRIPTION index e065312..8abadb5 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,13 +1,13 @@ Package: rfVarImpOOB Title: Unbiased Variable Importance for Random Forests -Version: 1.0 -Date: 2019-03-22 +Version: 1.0.1 +Date: 2020-10-17 Depends: R (>= 3.2.2), stats, randomForest Imports: ggplot2, binaryLogic, dplyr,titanic,prob,ggpubr,magrittr Suggests: knitr,rmarkdown Author: Markus Loecher Maintainer: Markus Loecher -Description: Computes a novel variable importance for random forests: Impurity reduction importance scores for out-of-bag (OOB) data complementing the existing inbag Gini importance, see also Strobl et al (2007) , Strobl et al (2007) and Breiman (2001) . +Description: Computes a novel variable importance for random forests: Impurity reduction importance scores for out-of-bag (OOB) data complementing the existing inbag Gini importance, see also . The Gini impurities for inbag and OOB data are combined in three different ways, after which the information gain is computed at each split. This gain is aggregated for each split variable in a tree and averaged across trees. License: GPL (>= 2) @@ -15,5 +15,5 @@ Repository: CRAN LazyData: true VignetteBuilder: knitr NeedsCompilation: no -Packaged: 2019-04-05 12:30:07 UTC; loecherm -Date/Publication: 2019-04-05 12:50:06 UTC +Packaged: 2020-10-18 08:40:08 UTC; loecherm +Date/Publication: 2020-10-18 09:00:02 UTC diff --git a/MD5 b/MD5 index 776ae32..903ce1b 100644 --- a/MD5 +++ b/MD5 @@ -1,15 +1,16 @@ -c4167422afb0ed7699160666f71d2b7b *DESCRIPTION +4e083e5da97590c90082400f8913d548 *DESCRIPTION 509fc06da5c156fe593494d5874b53e8 *NAMESPACE 33a36e8472ef3b252c258073f673f391 *R/InOutBags.R 2fc47a26497982c998a06f1ada0e0eb2 *R/LossFunctions.R 2fa6fc45bd8f81128480b1b00c754aa8 *R/allFuns.R 7bfd7a6b115d5a02d170991a1073fed3 *R/plotVI.R -d5e96726362a089b4f19cc0ec2b45833 *build/vignette.rds +f0058f80aa22f776f4da2e6f1312530f *build/vignette.rds 8a77c8837d56301cf3378a3cd52e3d95 *data/arabidopsis.rda 5c13dcaef01597d3acab3597dbc54f4f *data/titanic_train.rda +86cc7afab6f8123bd14e2d12e6fe139f *inst/CITATION 3e0aa94bd6ab65e84c92a1ff536614bf *inst/doc/rfVarImpOOB-vignette.R -28130f0bf8c1ecae98a27dfb97c1a054 *inst/doc/rfVarImpOOB-vignette.Rmd -345a7df24d1ccdeef30722a22d8bf0ea *inst/doc/rfVarImpOOB-vignette.html +b881710efa0af8764aa96546b857b740 *inst/doc/rfVarImpOOB-vignette.Rmd +e3a12326d3bd640fb3cd4aefb987b6b4 *inst/doc/rfVarImpOOB-vignette.html 125fdfbae3c353f3b5798f01764ab15c *man/Accuracy.Rd f76b2a8042c7fcb4836a44f70f8a1477 *man/GiniImportanceForest.Rd 3009f37a7f07b4e0c3f4dab9fd9d0a0b *man/GiniImportanceTree.Rd @@ -26,4 +27,4 @@ e4aa8b6a7ba1fd97145299f0d0315e70 *man/preorder2.Rd 714cb96bbcfc63bb1920a2852f3967bd *man/rfTitanic.Rd 84f751e594eec74bfc0ba3e288676529 *man/splitBag.Rd 6dc2cb14c3520377e957a7fa82d221b8 *man/titanic_train.Rd -28130f0bf8c1ecae98a27dfb97c1a054 *vignettes/rfVarImpOOB-vignette.Rmd +b881710efa0af8764aa96546b857b740 *vignettes/rfVarImpOOB-vignette.Rmd diff --git a/build/vignette.rds b/build/vignette.rds index 49aead2ebc0e5fedec80f7cb14636c8beeea0296..e4351fe34f3642232c47754a6d4a1ecb2dbbbbed 100644 GIT binary patch literal 231 zcmVJvg*gj(n+hSWxf2^r!sh)H^Qk7amdavfh?cQL>5vjC;Q z*~NjzO%KB3EQMf(IA>l2GZZCKG2w&8P8w@oR2*sGsJS`LYrm-vjb^rfRBgdxNVQU( zC0s!S#q+aSZ{(V!yMcR_$VyXVtdcQTJc6q4o~v&{x64%-8asW4zxuWLdoszKYKKAb hM-Oy4n!czXu~c8QA27w7P@=w=`vs^dJ{08v007~sXPf{4 literal 216 zcmV;}04M(+iwFP!0000028-ZgU|?WkU}j-vU}6R`nT3G_8xRWsF(U&D11FG{C`t=U zEb`1P@b`DpEz3;LOD!o$)eFi^K~u*U29;9?$t=l9MU$1pYDz{)ZVsA4G3?d>-6;%K z1yKzIEKEqcSo0EdQ;X4zVs}X`NX<(D3ID^cpYcDM77p*!%JTdom>v{0Y)~~UZka&u zLG6Y36~bW-aB+h&8H3Q>fMGt0`#Fp9%k^M>;047>1CaO+1n3@3&Pgmr_b69NVo9Q2 SS`kn`$lU;bf6>C%0RRBZ$X4wD diff --git a/inst/CITATION b/inst/CITATION new file mode 100644 index 0000000..5f1f6c7 --- /dev/null +++ b/inst/CITATION @@ -0,0 +1,18 @@ +bibentry(bibtype = "Article", + title = "Unbiased variable importance for random forests", + author = c(person(given = "Markus", + family = "Loecher")), + journal = "Communications in Statistics-Theory and Methods", + year = "2020", + volume = "0", + number = "0", + pages = "1--13", + doi = "10.1080/03610926.2020.1764042", + + header = "To rfVarImpOOB ranger in publications use:", + textVersion = + paste("Loecher, Markus (2020).", + "Unbiased variable importance for random forests", + "Communications in Statistics-Theory and Methods, 1-13.", + "10.1080/03610926.2020.1764042") +) diff --git a/inst/doc/rfVarImpOOB-vignette.Rmd b/inst/doc/rfVarImpOOB-vignette.Rmd index 244f666..7983bd4 100644 --- a/inst/doc/rfVarImpOOB-vignette.Rmd +++ b/inst/doc/rfVarImpOOB-vignette.Rmd @@ -4,7 +4,7 @@ author: "Markus Loecher" date: "`r Sys.Date()`" output: rmarkdown::html_vignette vignette: > - %\VignetteIndexEntry{Vignette Title} + %\VignetteIndexEntry{Unbiased Variable Importance } %\VignetteEngine{knitr::rmarkdown} %\VignetteEncoding{UTF-8} --- diff --git a/inst/doc/rfVarImpOOB-vignette.html b/inst/doc/rfVarImpOOB-vignette.html index 3ed2a02..182fa3c 100644 --- a/inst/doc/rfVarImpOOB-vignette.html +++ b/inst/doc/rfVarImpOOB-vignette.html @@ -7,59 +7,293 @@ + - + Variable Importance based on reduction of Gini on OOB - + + + + + - @@ -69,31 +303,633 @@

Variable Importance based on reduction of Gini on OOB

-

Markus Loecher

-

2019-04-05

+

Markus Loecher

+

2020-10-18

Variable importance in trees

Variable importance is not very well defined as a concept. Even for the case of a linear model with \(n\) observations, \(p\) variables and the standard \(n >> p\) situation, there is no theoretically defined variable importance metric in the sense of a parametric quantity that a variable importance estimator should try to estimate. Variable importance measures for random forests have been receiving increased attention in bioinformatics, for instance to select a subset of genetic markers relevant for the prediction of a certain disease. They also have been used as screening tools in important applications highlighting the need for reliable and well-understood feature importance measures.

-

The default choice in most software implementations of random forests is the mean decrease in impurity (MDI). The MDI of a feature is computed as a (weighted) mean of the individual trees’ improvement in the splitting criterion produced by each variable. A substantial shortcoming of this default measure is its evaluation on the in-bag samples which can lead to severe overfitting . It was also pointed out by Strobl et al. that the variable importance measures of Breiman’s original Random Forest method … are not reliable in situations where potential predictor variables vary in their scale of measurement or their number of categories. There have been multiple attempts at correcting the well understood bias of the Gini impurity measure both as a split cirterion as well as a contributor to importance scores, each one coming from a different perspective. Strobl et al. derive the exact distribution of the maximally selected Gini gain along with their resulting p-values by means of a combinatorial approach. Shi et al. suggest a solution to the bias for the case of regression trees as well as binary classification trees which is also based on P-values. Several authors argue that the criterion for split variable and split point selection should be separated.

+

The default choice in most software implementations of random forests is the mean decrease in impurity (MDI). The MDI of a feature is computed as a (weighted) mean of the individual trees’ improvement in the splitting criterion produced by each variable. A substantial shortcoming of this default measure is its evaluation on the in-bag samples which can lead to severe overfitting . It was also pointed out by Strobl et al. that the variable importance measures of Breiman’s original Random Forest method … are not reliable in situations where potential predictor variables vary in their scale of measurement or their number of categories. There have been multiple attempts at correcting the well understood bias of the Gini impurity measure both as a split cirterion as well as a contributor to importance scores, each one coming from a different perspective. Strobl et al. derive the exact distribution of the maximally selected Gini gain along with their resulting p-values by means of a combinatorial approach. Shi et al. suggest a solution to the bias for the case of regression trees as well as binary classification trees which is also based on P-values. Several authors argue that the criterion for split variable and split point selection should be separated.

We use the well known titanic data set to illustrate the perils of putting too much faith into the Gini importance which is based entirely on training data - not on OOB samples - and makes no attempt to discount impurity decreases in deep trees that are pretty much frivolous and will not survive in a validation set. In the following model we include passengerID as a feature along with the more reasonable Age, Sex and Pclass.

Penalized Gini Impurity applied to Titanic data

The Figure below show both measures of variable importance and (maybe?) surprisingly passengerID turns out to be ranked number \(3\) for the Gini importance (MDI). This troubling result is robust to random shuffling of the ID.

-
naRows = is.na(titanic_train$Age)
-data2=titanic_train[!naRows,]
-RF =randomForest(Survived ~ Age + Sex + Pclass + PassengerId, data=data2, ntree=50,importance=TRUE,mtry=2, keep.inbag=TRUE)
-
if (is.factor(data2$Survived)) data2$Survived = as.numeric(data2$Survived)-1
-VI_PMDI3 = GiniImportanceForest(RF, data2,score="PMDI22",Predictor=mean)
-plotVI2(VI_PMDI3, score="PMDI22", decreasing = TRUE)
-

+ +
if (is.factor(data2$Survived)) data2$Survived = as.numeric(data2$Survived)-1
+VI_PMDI3 = GiniImportanceForest(RF, data2,score="PMDI22",Predictor=mean)
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+
+#> Warning: Factor `split var` contains implicit NA, consider using
+#> `forcats::fct_explicit_na`
+plotVI2(VI_PMDI3, score="PMDI22", decreasing = TRUE)
+

+ + +