Merge branch 'master' into 656-using-patch-miner-without-label-header…

…-results-in-failure

GANDLF/optimizers/__init__.py

@@ -9,6 +9,7 @@
     adadelta,
     adagrad,
     rmsprop,
+    radam,
 )
 
 global_optimizer_dict = {
@@ -22,6 +23,7 @@
     "adadelta": adadelta,
     "adagrad": adagrad,
     "rmsprop": rmsprop,
+    "radam": radam,
 }
 
 

GANDLF/optimizers/wrap_torch.py

@@ -9,6 +9,7 @@
     Adadelta,
     Adagrad,
     RMSprop,
+    RAdam
 )
 
 
@@ -223,3 +224,23 @@ def rmsprop(parameters):
         weight_decay=parameters["optimizer"].get("weight_decay", 0),
     )
 
+def radam(parameters):
+    """
+    Creates a RAdam optimizer from the PyTorch `torch.optim` module using the input parameters. 
+    
+    Args: 
+        parameters (dict): A dictionary containing the input parameters for the optimizer.
+        
+    Returns: 
+        optimizer (torch.optim.RAdam): A RAdam optimizer.
+    """
+    # Create the optimizer using the input parameters
+    return RAdam(
+        parameters["model_parameters"],
+        lr=parameters.get("learning_rate"),
+        betas=parameters["optimizer"].get("betas", (0.9, 0.999)),
+        eps=parameters["optimizer"].get("eps", 1e-8),
+        weight_decay=parameters["optimizer"].get("weight_decay", 0),
+        foreach=parameters["optimizer"].get("foreach", None),
+    )
+