Skip to content

Commit

Permalink
Draft version 0.3.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@FBerding
FBerding committed Oct 7, 2023
1 parent 73f60fe commit ae03b1e
Show file tree
Hide file tree
Showing 23 changed files with 235 additions and 120 deletions.
2 changes: 1 addition & 1 deletion DESCRIPTION
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
Type: Package
Package: aifeducation
Title: Artificial Intelligence for Education
Version: 0.2.1.9003
Version: 0.3.0.9000
Authors@R: c(
person("Berding", "Florian", , "florian.berding@uni-hamburg.de", role = c("aut", "cre"),
comment = c(ORCID = "0000-0002-3593-1695")),
Expand Down
1 change: 1 addition & 0 deletions NAMESPACE
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ export(set_config_cpu_only)
export(set_config_gpu_low_memory)
export(set_config_os_environ_logger)
export(set_config_tf_logger)
export(set_transformers_logger)
export(train_tune_bert_model)
export(train_tune_deberta_v2_model)
export(train_tune_funnel_model)
Expand Down
118 changes: 64 additions & 54 deletions R/aux_fct.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,9 @@ return(TRUE)
#'
#'@param true_values \code{factor} containing the true labels/categories.
#'@param predicted_values \code{factor} containing the predicted labels/categories.
#'@return Returns a \code{vector} with the following reliability measures:
#'@param return_names_only \code{bool} If \code{TRUE} returns only the names
#'of the resulting vector. Use {FALSE} to request computation of the values.
#'@return If \code{return_names_only=FALSE} returns a \code{vector} with the following reliability measures:
#'#'\itemize{
#'\item{\strong{iota_index: }}{Iota Index from the Iota Reliability Concept Version 2.}
#'\item{\strong{min_iota2: }}{Minimal Iota from Iota Reliability Concept Version 2.}
Expand All @@ -178,14 +180,14 @@ return(TRUE)
#'\item{\strong{gwet_ac: }}{Gwet's AC1/AC2 agreement coefficient.}
#'}
#'
#'@return If \code{return_names_only=TRUE} returns only the names of the vector elements.
#'
#'@family Auxiliary Functions
#'
#'@export
get_coder_metrics<-function(true_values,
predicted_values){
val_res=iotarelr::check_new_rater(true_values = true_values,
assigned_values = predicted_values,
free_aem = TRUE)
get_coder_metrics<-function(true_values=NULL,
predicted_values=NULL,
return_names_only=FALSE){

metric_names=c("iota_index",
"min_iota2",
Expand All @@ -208,54 +210,62 @@ val_res=iotarelr::check_new_rater(true_values = true_values,
metric_values=vector(length = length(metric_names))
names(metric_values)=metric_names

val_res=iotarelr::check_new_rater(true_values = true_values,
assigned_values = predicted_values,
free_aem = FALSE)
val_res_free=iotarelr::check_new_rater(true_values = true_values,
assigned_values = predicted_values,
free_aem = TRUE)

metric_values["iota_index"]=val_res$scale_level$iota_index

metric_values["min_iota2"]=min(val_res_free$categorical_level$raw_estimates$iota)
metric_values["avg_iota2"]=mean(val_res_free$categorical_level$raw_estimates$iota)
metric_values["max_iota2"]=max(val_res_free$categorical_level$raw_estimates$iota)

metric_values["min_alpha"]=min(val_res_free$categorical_level$raw_estimates$alpha_reliability)
metric_values["avg_alpha"]=mean(val_res_free$categorical_level$raw_estimates$alpha_reliability)
metric_values["max_alpha"]=max(val_res_free$categorical_level$raw_estimates$alpha_reliability)

metric_values["static_iota_index"]=val_res$scale_level$iota_index_d4
metric_values["dynamic_iota_index"]=val_res$scale_level$iota_index_dyn2

metric_values["kalpha_nominal"]=irr::kripp.alpha(x=rbind(true_values,predicted_values),
method = "nominal")$value
metric_values["kalpha_ordinal"]=irr::kripp.alpha(x=rbind(true_values,predicted_values),
method = "ordinal")$value

metric_values["kendall"]=irr::kendall(ratings=cbind(true_values,predicted_values),
correct=TRUE)$value

metric_values["kappa2_unweighted"]=irr::kappa2(ratings=cbind(true_values,predicted_values),
weight = "unweighted",
sort.levels = FALSE)$value
metric_values["kappa2_equal_weighted"]=irr::kappa2(ratings=cbind(true_values,predicted_values),
weight = "equal",
sort.levels = FALSE)$value
metric_values["kappa2_squared_weighted"]=irr::kappa2(ratings=cbind(true_values,predicted_values),
weight = "squared",
sort.levels = FALSE)$value

metric_values["kappa_fleiss"]=irr::kappam.fleiss(ratings=cbind(true_values,predicted_values),
exact = FALSE,
detail = FALSE)$value

metric_values["percentage_agreement"]=irr::agree(ratings=cbind(true_values,predicted_values),
tolerance = 0)$value/100

metric_values["gwet_ac"]=irrCAC::gwet.ac1.raw(ratings=cbind(true_values,predicted_values))$est$coeff.val

return(metric_values)
if(return_names_only==TRUE){
return(metric_names)
} else {
val_res=iotarelr::check_new_rater(true_values = true_values,
assigned_values = predicted_values,
free_aem = TRUE)

val_res=iotarelr::check_new_rater(true_values = true_values,
assigned_values = predicted_values,
free_aem = FALSE)
val_res_free=iotarelr::check_new_rater(true_values = true_values,
assigned_values = predicted_values,
free_aem = TRUE)

metric_values["iota_index"]=val_res$scale_level$iota_index

metric_values["min_iota2"]=min(val_res_free$categorical_level$raw_estimates$iota)
metric_values["avg_iota2"]=mean(val_res_free$categorical_level$raw_estimates$iota)
metric_values["max_iota2"]=max(val_res_free$categorical_level$raw_estimates$iota)

metric_values["min_alpha"]=min(val_res_free$categorical_level$raw_estimates$alpha_reliability)
metric_values["avg_alpha"]=mean(val_res_free$categorical_level$raw_estimates$alpha_reliability)
metric_values["max_alpha"]=max(val_res_free$categorical_level$raw_estimates$alpha_reliability)

metric_values["static_iota_index"]=val_res$scale_level$iota_index_d4
metric_values["dynamic_iota_index"]=val_res$scale_level$iota_index_dyn2

metric_values["kalpha_nominal"]=irr::kripp.alpha(x=rbind(true_values,predicted_values),
method = "nominal")$value
metric_values["kalpha_ordinal"]=irr::kripp.alpha(x=rbind(true_values,predicted_values),
method = "ordinal")$value

metric_values["kendall"]=irr::kendall(ratings=cbind(true_values,predicted_values),
correct=TRUE)$value

metric_values["kappa2_unweighted"]=irr::kappa2(ratings=cbind(true_values,predicted_values),
weight = "unweighted",
sort.levels = FALSE)$value
metric_values["kappa2_equal_weighted"]=irr::kappa2(ratings=cbind(true_values,predicted_values),
weight = "equal",
sort.levels = FALSE)$value
metric_values["kappa2_squared_weighted"]=irr::kappa2(ratings=cbind(true_values,predicted_values),
weight = "squared",
sort.levels = FALSE)$value

metric_values["kappa_fleiss"]=irr::kappam.fleiss(ratings=cbind(true_values,predicted_values),
exact = FALSE,
detail = FALSE)$value

metric_values["percentage_agreement"]=irr::agree(ratings=cbind(true_values,predicted_values),
tolerance = 0)$value/100

metric_values["gwet_ac"]=irrCAC::gwet.ac1.raw(ratings=cbind(true_values,predicted_values))$est$coeff.val

return(metric_values)
}
}

#------------------------------------------------------------------------------
Expand Down
26 changes: 25 additions & 1 deletion R/install_and_config.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ set_config_tf_logger<-function(level="ERROR"){
#'This function changes the level for logging information with 'tensorflow' via
#'the os environment. This function must be called before importing 'tensorflow'.
#'
#'@param level \code{string} Minimal level that should be printed to console. Five
#'@param level \code{string} Minimal level that should be printed to console. Four
#'levels are available: INFO, WARNING, ERROR and NONE.
#'@return This function does not return anything. It is used for its
#'side effects.
Expand All @@ -204,6 +204,30 @@ set_config_os_environ_logger<-function(level="ERROR"){
os$environ$setdefault("TF_CPP_MIN_LOG_LEVEL","2")
}

#'Sets the level for logging information of the 'transformers' library.
#'
#'This function changes the level for logging information of the 'transformers' library.
#'It influences the output printed to console for creating and training transformer models as well as
#'\link{TextEmbeddingModel}s.
#'
#'@param level \code{string} Minimal level that should be printed to console. Four
#'levels are available: INFO, WARNING, ERROR and DEBUG
#'@return This function does not return anything. It is used for its
#'side effects.
#'@family Installation and Configuration
#'@export
set_transformers_logger<-function(level="ERROR"){
if(level=="ERROR"){
transformers$utils$logging$set_verbosity_error()
} else if (level=="WARNING"){
transformers$utils$logging$set_verbosity_warning()
} else if (level=="INFO"){
transformers$utils$logging$set_verbosity_info()
} else if(level=="DEBUG"){
transformers$utils$logging$set_verbosity_debug()
}
}

#'R6 class for settting the global machine learning framework.
#'
#'R6 class for setting the global machine learning framework to 'PyTorch' or
Expand Down
26 changes: 7 additions & 19 deletions R/te_classifier_neuralnet_model.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -789,32 +789,20 @@ TextEmbeddingClassifierNeuralNet<-R6::R6Class(
data_bsc_test=data_bsc_train

#Initializing Objects for Saving Performance
metric_names=get_coder_metrics(
true_values=NULL,
predicted_values=NULL,
return_names_only=TRUE)

test_metric=array(dim=c(folds$n_folds,
4,
18),
length(metric_names)),
dimnames = list(iterations=NULL,
steps=c("Baseline",
"BSC",
"BPL",
"Final"),
metrics=c("iota_index",
"min_iota2",
"avg_iota2",
"max_iota2",
"min_alpha",
"avg_alpha",
"max_alpha",
"static_iota_index",
"dynamic_iota_index",
"kalpha_nominal",
"kalpha_ordinal",
"kendall",
"kappa2_unweighted",
"kappa2_equal_weighted",
"kappa2_squared_weighted",
"kappa_fleiss",
"percentage_agreement",
"gwet_ac")))
metrics=metric_names))
iota_objects_start=NULL
iota_objects_end=NULL
iota_objects_start_free=NULL
Expand Down

0 comments on commit ae03b1e

Please sign in to comment.