Skip to content

[SYSTEMDS-3378] Python API autogenerator default values #1615

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Baunsgaard wants to merge 4 commits into from
Closed

[SYSTEMDS-3378] Python API autogenerator default values #1615

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
363524c
[MINOR] Builtin Modifications for consistency
Baunsgaard May 16, 2022
aae814d
[SYSTEMDS-3378] Python API auto generator default values
Baunsgaard May 16, 2022
7ea9465
Fix Builtins lambda argument
Baunsgaard May 16, 2022
5fc18df
Rename lambda to reg in builtins
Baunsgaard May 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions scripts/algorithms/l2-svm.dml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
fmt = ifdef($fmt, "text")
intercept = ifdef($icpt, FALSE)
epsilon = ifdef($tol, 0.001)
lambda = ifdef($reg, 1.0)
reg = ifdef($reg, 1.0)
maxIterations = ifdef($maxiter, 100)
verbose = ifdef($verbose, FALSE)

Expand All @@ -62,7 +62,7 @@ negative_label = min(Y)
dimensions = ncol(X)

w = l2svm(X=X, Y=Y, intercept=intercept,
epsilon=epsilon, lambda=lambda,
epsilon=epsilon, reg=reg,
maxIterations=maxIterations,
verbose=verbose)

Expand Down
20 changes: 10 additions & 10 deletions scripts/builtin/als.dml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,18 +29,18 @@
# ----------------------------------------------------------------------------------------------------------------------
# X Matrix[Double] --- Location to read the input matrix X to be factorized
# rank Integer 10 Rank of the factorization
# reg String "L2" Regularization:
# regType String "L2" Regularization:
# "L2" = L2 regularization;
# f (U, V) = 0.5 * sum (W * (U %*% V - X) ^ 2)
# + 0.5 * lambda * (sum (U ^ 2) + sum (V ^ 2))
# + 0.5 * reg * (sum (U ^ 2) + sum (V ^ 2))
# "wL2" = weighted L2 regularization
# f (U, V) = 0.5 * sum (W * (U %*% V - X) ^ 2)
# + 0.5 * lambda * (sum (U ^ 2 * row_nonzeros)
# + 0.5 * reg * (sum (U ^ 2 * row_nonzeros)
# + sum (V ^ 2 * col_nonzeros))
# lambda Double 0.000001 Regularization parameter, no regularization if 0.0
# maxi Integer 50 Maximum number of iterations
# check Boolean TRUE Check for convergence after every iteration, i.e., updating U and V once
# thr Double 0.0001 Assuming check is set to TRUE, the algorithm stops and convergence is declared
# reg Double 0.000001 Regularization parameter, no regularization if 0.0
# maxi Integer 50 Maximum number of iterations
# check Boolean TRUE Check for convergence after every iteration, i.e., updating U and V once
# thr Double 0.0001 Assuming check is set to TRUE, the algorithm stops and convergence is declared
# if the decrease in loss in any two consecutive iterations falls below this threshold;
# if check is FALSE thr is ignored
# ----------------------------------------------------------------------------------------------------------------------
Expand All @@ -53,15 +53,15 @@
# V Matrix An m x r matrix where r is the factorization rank
# ----------------------------------------------------------------------------------------------------------------------

m_als = function(Matrix[Double] X, Integer rank = 10, String reg = "L2", Double lambda = 0.000001,
m_als = function(Matrix[Double] X, Integer rank = 10, String regType = "L2", Double reg = 0.000001,
Integer maxi = 50, Boolean check = TRUE, Double thr = 0.0001, Boolean verbose = TRUE)
return (Matrix[Double] U, Matrix[Double] V)
{
N = 10000; # for large problems, use scalable alsCG
if( reg != "L2" | nrow(X) > N | ncol(X) > N )
[U, V] = alsCG(X=X, rank=rank, reg=reg, lambda=lambda,
[U, V] = alsCG(X=X, rank=rank, regType=regType, reg=reg,
maxi=maxi, check=check, thr=thr, verbose=verbose);
else
[U, V] = alsDS(X=X, rank=rank, lambda=lambda, maxi=maxi,
[U, V] = alsDS(X=X, rank=rank, reg=reg, maxi=maxi,
check=check, thr=thr, verbose=verbose);
}
42 changes: 21 additions & 21 deletions scripts/builtin/alsCG.dml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,18 +29,18 @@
# ----------------------------------------------------------------------------------------------------------------------
# X Matrix[Double] --- Location to read the input matrix X to be factorized
# rank Integer 10 Rank of the factorization
# reg String "L2" Regularization:
# regType String "L2" Regularization:
# "L2" = L2 regularization;
# f (U, V) = 0.5 * sum (W * (U %*% V - X) ^ 2)
# + 0.5 * lambda * (sum (U ^ 2) + sum (V ^ 2))
# + 0.5 * reg * (sum (U ^ 2) + sum (V ^ 2))
# "wL2" = weighted L2 regularization
# f (U, V) = 0.5 * sum (W * (U %*% V - X) ^ 2)
# + 0.5 * lambda * (sum (U ^ 2 * row_nonzeros)
# + 0.5 * reg * (sum (U ^ 2 * row_nonzeros)
# + sum (V ^ 2 * col_nonzeros))
# lambda Double 0.000001 Regularization parameter, no regularization if 0.0
# maxi Integer 50 Maximum number of iterations
# check Boolean TRUE Check for convergence after every iteration, i.e., updating U and V once
# thr Double 0.0001 Assuming check is set to TRUE, the algorithm stops and convergence is declared
# reg Double 0.000001 Regularization parameter, no regularization if 0.0
# maxi Integer 50 Maximum number of iterations
# check Boolean TRUE Check for convergence after every iteration, i.e., updating U and V once
# thr Double 0.0001 Assuming check is set to TRUE, the algorithm stops and convergence is declared
# if the decrease in loss in any two consecutive iterations falls below this threshold;
# if check is FALSE thr is ignored
# ----------------------------------------------------------------------------------------------------------------------
Expand All @@ -53,7 +53,7 @@
# V Matrix[Double] An m x r matrix where r is the factorization rank
# ----------------------------------------------------------------------------------------------------------------------

m_alsCG = function(Matrix[Double] X, Integer rank = 10, String reg = "L2", Double lambda = 0.000001, Integer maxi = 50,
m_alsCG = function(Matrix[Double] X, Integer rank = 10, String regType = "L2", Double reg = 0.000001, Integer maxi = 50,
Boolean check = TRUE, Double thr = 0.0001, Boolean verbose = TRUE)
return (Matrix[Double] U, Matrix[Double] V)
{
Expand All @@ -73,26 +73,26 @@ m_alsCG = function(Matrix[Double] X, Integer rank = 10, String reg = "L2", Doubl
# check for regularization
row_nonzeros = matrix(0,rows=1,cols=1);
col_nonzeros = matrix(0,rows=1,cols=1);
if( reg == "L2" ) {
if( regType == "L2" ) {
# Loss Function with L2:
# f (U, V) = 0.5 * sum (W * (U %*% V - X) ^ 2)
# + 0.5 * lambda * (sum (U ^ 2) + sum (V ^ 2))
# + 0.5 * reg * (sum (U ^ 2) + sum (V ^ 2))
if( verbose )
print ("BEGIN ALS-CG SCRIPT WITH NONZERO SQUARED LOSS + L2 WITH LAMBDA - " + lambda);
print ("BEGIN ALS-CG SCRIPT WITH NONZERO SQUARED LOSS + L2 WITH REG - " + reg);
row_nonzeros = matrix(1, nrow(W), 1);
col_nonzeros = matrix(1, ncol(W), 1);
}
else if( reg == "wL2" ) {
else if( regType == "wL2" ) {
# Loss Function with weighted L2:
# f (U, V) = 0.5 * sum (W * (U %*% V - X) ^ 2)
# + 0.5 * lambda * (sum (U ^ 2 * row_nonzeros) + sum (V ^ 2 * col_nonzeros))
# + 0.5 * reg * (sum (U ^ 2 * row_nonzeros) + sum (V ^ 2 * col_nonzeros))
if( verbose )
print ("BEGIN ALS-CG SCRIPT WITH NONZERO SQUARED LOSS + WEIGHTED L2 WITH LAMBDA - " + lambda);
print ("BEGIN ALS-CG SCRIPT WITH NONZERO SQUARED LOSS + WEIGHTED L2 WITH REG - " + reg);
row_nonzeros = rowSums(W);
col_nonzeros = t(colSums(W));
}
else {
stop ("wrong regularization! " + reg);
stop ("wrong regularization! " + regType);
}

is_U = TRUE; # start optimizing U, alternated
Expand All @@ -101,7 +101,7 @@ m_alsCG = function(Matrix[Double] X, Integer rank = 10, String reg = "L2", Doubl
loss_init = 0.0; # only used if check is TRUE
if( check ) {
loss_init = 0.5 * sum( (X != 0) * (U %*% t(V) - X) ^ 2);
loss_init = loss_init + 0.5 * lambda * (sum (U ^ 2 * row_nonzeros) + sum (V ^ 2 * col_nonzeros));
loss_init = loss_init + 0.5 * reg * (sum (U ^ 2 * row_nonzeros) + sum (V ^ 2 * col_nonzeros));
if( verbose )
print ("----- Initial train loss: " + loss_init + " -----");
}
Expand All @@ -111,9 +111,9 @@ m_alsCG = function(Matrix[Double] X, Integer rank = 10, String reg = "L2", Doubl
while( as.integer(it/2) < max_iter & ! converged ) {
it = it + 1;
if( is_U )
G = ((X != 0) * (U %*% t(V) - X)) %*% V + lambda * U * row_nonzeros;
G = ((X != 0) * (U %*% t(V) - X)) %*% V + reg * U * row_nonzeros;
else
G = t(t(U) %*% ((X != 0) * (U %*% t(V) - X))) + lambda * V * col_nonzeros;
G = t(t(U) %*% ((X != 0) * (U %*% t(V) - X))) + reg * V * col_nonzeros;

R = -G;
S = R;
Expand All @@ -124,12 +124,12 @@ m_alsCG = function(Matrix[Double] X, Integer rank = 10, String reg = "L2", Doubl
tt = 0.000000001;
while( norm_R2 > tt * norm_G2 & inneriter <= maxinneriter ) {
if( is_U ) {
HS = (W * (S %*% t(V))) %*% V + lambda * S * row_nonzeros;
HS = (W * (S %*% t(V))) %*% V + reg * S * row_nonzeros;
alpha = norm_R2 / sum (S * HS);
U = U + alpha * S; # OK since U is not used in HS
}
else {
HS = t(t(U) %*% (W * (U %*% t(S)))) + lambda * S * col_nonzeros;
HS = t(t(U) %*% (W * (U %*% t(S)))) + reg * S * col_nonzeros;
alpha = norm_R2 / sum (S * HS);
V = V + alpha * S; # OK since V is not used in HS
}
Expand All @@ -146,7 +146,7 @@ m_alsCG = function(Matrix[Double] X, Integer rank = 10, String reg = "L2", Doubl
# check for convergence
if( check & (it%%2 == 0) ) {
loss_cur = 0.5 * sum( (X != 0) * (U %*% t(V) - X) ^ 2);
loss_cur = loss_cur + 0.5 * lambda * (sum (U ^ 2 * row_nonzeros) + sum (V ^ 2 * col_nonzeros));
loss_cur = loss_cur + 0.5 * reg * (sum (U ^ 2 * row_nonzeros) + sum (V ^ 2 * col_nonzeros));

loss_dec = (loss_init - loss_cur) / loss_init;
if( verbose )
Expand Down
12 changes: 6 additions & 6 deletions scripts/builtin/alsDS.dml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
# ----------------------------------------------------------------------------------------------------------------------
# X Matrix[Double] --- Location to read the input matrix V to be factorized
# rank Integer 10 Rank of the factorization
# lambda Double 0.000001 Regularization parameter, no regularization if 0.0
# reg Double 0.000001 Regularization parameter, no regularization if 0.0
# maxi Integer 50 Maximum number of iterations
# check Boolean FALSE Check for convergence after every iteration, i.e., updating L and R once
# thr Double 0.0001 Assuming check is set to TRUE, the algorithm stops and convergence is declared
Expand All @@ -46,7 +46,7 @@
# V Matrix[Double] An m x r matrix where r is the factorization rank
# ----------------------------------------------------------------------------------------------------------------------

m_alsDS = function(Matrix[Double] X, Integer rank = 10, Double lambda = 0.000001,
m_alsDS = function(Matrix[Double] X, Integer rank = 10, Double reg = 0.000001,
Integer maxi = 50, Boolean check = FALSE, Double thr = 0.0001, Boolean verbose = TRUE)
return (Matrix[Double] U, Matrix[Double] V)
{
Expand Down Expand Up @@ -92,17 +92,17 @@ m_alsDS = function(Matrix[Double] X, Integer rank = 10, Double lambda = 0.000001

# check for regularization
if ( verbose )
print ("BEGIN ALS SCRIPT WITH NONZERO SQUARED LOSS + L2 WITH LAMBDA - " + lambda);
print ("BEGIN ALS SCRIPT WITH NONZERO SQUARED LOSS + L2 WITH REG - " + reg);

loss_init = 0.0; # only used if check is TRUE
if (check) {
loss_init = sum (X_nonzero_ind * (X - (U %*% t(V)))^2)
+ lambda * (sum ((U^2) * row_nonzeros) + sum ((V^2) * col_nonzeros));
+ reg * (sum ((U^2) * row_nonzeros) + sum ((V^2) * col_nonzeros));
if( verbose )
print ("----- Initial train loss: " + loss_init + " -----");
}

lambda_I = diag (matrix (lambda, rows = r, cols = 1));
lambda_I = diag (matrix (reg, rows = r, cols = 1));
it = 0;
converged = FALSE;
while ((it < max_iter) & (!converged)) {
Expand All @@ -126,7 +126,7 @@ m_alsDS = function(Matrix[Double] X, Integer rank = 10, Double lambda = 0.000001
# check for convergence
if (check) {
loss_cur = sum (X_nonzero_ind * (X - (U %*% t(V)))^2)
+ lambda * (sum ((U^2) * row_nonzeros) + sum ((V^2) * col_nonzeros));
+ reg * (sum ((U^2) * row_nonzeros) + sum ((V^2) * col_nonzeros));
loss_dec = (loss_init - loss_cur) / loss_init;
if( verbose )
print ("Train loss at iteration (X) " + it + ": " + loss_cur + " loss-dec " + loss_dec);
Expand Down
1 change: 0 additions & 1 deletion scripts/builtin/bandit.dml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@
m_bandit = function(Matrix[Double] X_train, Matrix[Double] Y_train, Matrix[Double] X_test, Matrix[Double] Y_test, List[Unknown] metaList,
String evaluationFunc, Matrix[Double] evalFunHp, Frame[Unknown] lp, Matrix[Double] lpHp, Frame[Unknown] primitives, Frame[Unknown] param, Integer k = 3,
Integer R=50, Double baseLineScore, Boolean cv, Integer cvk = 2, Double ref = 0, Integer seed = -1, Boolean enablePruning = FALSE, Boolean verbose = TRUE)
# return(Boolean perf)
return (Frame[Unknown] bestPipeline, Matrix[Double] bestHyperparams, Matrix[Double] bestAccuracy, Frame[String] applyFunc)
{
print("Starting optimizer")
Expand Down
2 changes: 1 addition & 1 deletion scripts/builtin/cox.dml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@
# ----------------------------------------------------------------------------------------------------------------------

m_cox = function(Matrix[Double] X, Matrix[Double] TE, Matrix[Double] F, Matrix[Double] R,
Double alpha = 0.05, Double tol = 0.000001, Int moi = 100, Int mii = 0)
Double alpha = 0.05, Double tol = 0.000001, Integer moi = 100, Integer mii = 0)
return (Matrix[Double] M, Matrix[Double] S, Matrix[Double] T, Matrix[Double] COV, Matrix[Double] RT, Matrix[Double] XO) {

X_orig = X;
Expand Down
14 changes: 7 additions & 7 deletions scripts/builtin/l2svm.dml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
# intercept Boolean False No Intercept ( If set to TRUE then a constant bias column is added to X)
# epsilon Double 0.001 Procedure terminates early if the reduction in objective function value is less
# than epsilon (tolerance) times the initial objective function value.
# lambda Double 1.0 Regularization parameter (lambda) for L2 regularization
# reg Double 1.0 Regularization parameter (reg) for L2 regularization
# maxIterations Int 100 Maximum number of conjugate gradient iterations
# maxii Int 20 -
# verbose Boolean FALSE Set to true if one wants print statements updating on loss.
Expand All @@ -46,7 +46,7 @@
# ----------------------------------------------------------------------------------------------------------------------

m_l2svm = function(Matrix[Double] X, Matrix[Double] Y, Boolean intercept = FALSE,
Double epsilon = 0.001, Double lambda = 1, Integer maxIterations = 100,
Double epsilon = 0.001, Double reg = 1, Integer maxIterations = 100,
Integer maxii = 20, Boolean verbose = FALSE, Integer columnId = -1)
return(Matrix[Double] model)
{
Expand All @@ -55,7 +55,7 @@ m_l2svm = function(Matrix[Double] X, Matrix[Double] Y, Boolean intercept = FALSE
stop("L2SVM: Stopping due to invalid inputs: Not possible to learn a binary class classifier without at least 2 rows")
if(epsilon < 0)
stop("L2SVM: Stopping due to invalid argument: Tolerance (tol) must be non-negative")
if(lambda < 0)
if(reg < 0)
stop("L2SVM: Stopping due to invalid argument: Regularization constant (reg) must be non-negative")
if(maxIterations < 1)
stop("L2SVM: Stopping due to invalid argument: Maximum iterations should be a positive integer")
Expand Down Expand Up @@ -106,8 +106,8 @@ m_l2svm = function(Matrix[Double] X, Matrix[Double] Y, Boolean intercept = FALSE
# minimizing primal obj along direction s
step_sz = 0
Xd = X %*% s
wd = lambda * sum(w * s)
dd = lambda * sum(s * s)
wd = reg * sum(w * s)
dd = reg * sum(s * s)
continue1 = TRUE
iiter = 0
while(continue1 & iiter < maxii){
Expand All @@ -129,8 +129,8 @@ m_l2svm = function(Matrix[Double] X, Matrix[Double] Y, Boolean intercept = FALSE
out = 1 - Y * Xw
sv = (out > 0)
out = sv * out
obj = 0.5 * sum(out * out) + lambda/2 * sum(w * w)
g_new = t(X) %*% (out * Y) - lambda * w
obj = 0.5 * sum(out * out) + reg/2 * sum(w * w)
g_new = t(X) %*% (out * Y) - reg * w

if(verbose) {
colstr = ifelse(columnId!=-1, ", Col:"+columnId + " ,", " ,")
Expand Down
16 changes: 8 additions & 8 deletions scripts/builtin/lenetTrain.dml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
# lr Double 0.01 Learning rate
# mu Double 0.9 Momentum value
# decay Double 0.95 Learning rate decay
# lambda Double 5e-04 Regularization strength
# reg Double 5e-04 Regularization strength
# seed Integer -1 Seed for model initialization
# verbose Boolean FALSE Flag indicates if function should print to stdout
# ----------------------------------------------------------------------------------------------------------------------
Expand All @@ -64,7 +64,7 @@ source("nn/layers/lenetForwardPass.dml") as lenet_fw

m_lenetTrain = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] X_val,
Matrix[Double] Y_val, Integer C, Integer Hin, Integer Win, Integer batch_size=64,
Integer epochs=20, Double lr=0.01, Double mu=0.9, Double decay=0.95, Double lambda=5e-04,
Integer epochs=20, Double lr=0.01, Double mu=0.9, Double decay=0.95, Double reg=5e-04,
Boolean verbose=FALSE, Integer seed=-1)
return (List[unknown] model)
{
Expand Down Expand Up @@ -126,7 +126,7 @@ m_lenetTrain = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] X_val

# Compute data backward pass
[dW1, db1, dW2, db2, dW3, db3, dW4, db4] = feed_backward(
X_batch, C, Hin, Win, lambda, model, dprobs, cache)
X_batch, C, Hin, Win, reg, model, dprobs, cache)

# Optimize with SGD w/ Nesterov momentum
[W1, vW1] = sgd_nesterov::update(W1, dW1, lr, mu, vW1)
Expand All @@ -153,7 +153,7 @@ m_lenetTrain = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] X_val
}

feed_backward = function(Matrix[Double] X, Integer C, Integer Hin, Integer Win,
Double lambda,list[unknown] model, matrix[Double] dprobs, list[unknown] cache)
Double reg,list[unknown] model, matrix[Double] dprobs, list[unknown] cache)
return (Matrix[Double] dW1, Matrix[Double] db1,
Matrix[Double] dW2, Matrix[Double] db2,
Matrix[Double] dW3, Matrix[Double] db3,
Expand Down Expand Up @@ -193,10 +193,10 @@ feed_backward = function(Matrix[Double] X, Integer C, Integer Hin, Integer Win,
X, as.matrix(model["W1"]), as.matrix(model["b1"]), C, Hin, Win, Hf, Wf, stride, stride, pad, pad)

# Compute regularization backward pass
dW1_reg = l2_reg::backward(as.matrix(model["W1"]), lambda)
dW2_reg = l2_reg::backward(as.matrix(model["W2"]), lambda)
dW3_reg = l2_reg::backward(as.matrix(model["W3"]), lambda)
dW4_reg = l2_reg::backward(as.matrix(model["W4"]), lambda)
dW1_reg = l2_reg::backward(as.matrix(model["W1"]), reg)
dW2_reg = l2_reg::backward(as.matrix(model["W2"]), reg)
dW3_reg = l2_reg::backward(as.matrix(model["W3"]), reg)
dW4_reg = l2_reg::backward(as.matrix(model["W4"]), reg)
dW1 = dW1 + dW1_reg
dW2 = dW2 + dW2_reg
dW3 = dW3 + dW3_reg
Expand Down
Loading