From 29232615bbf0b1ad9ded6725b74de84f7593d92b Mon Sep 17 00:00:00 2001 From: Atefeh Asayesh Date: Tue, 8 Jun 2021 10:52:19 +0200 Subject: [PATCH 1/3] residency hospital matching Signed-off-by: Atefeh Asayesh --- .../validation/ValidateLicAndNotice.java | 1 - scripts/builtin/residencyMatchMain.dml | 194 ++++++++++++++++++ src/assembly/extra/LICENSE | 51 ----- .../org/apache/sysds/common/Builtins.java | 83 ++++---- .../builtin/BuiltinResidencyMatchTest.java | 95 +++++++++ .../functions/builtin/residencymatch.dml | 10 + 6 files changed, 337 insertions(+), 97 deletions(-) create mode 100644 scripts/builtin/residencyMatchMain.dml create mode 100644 src/test/java/org/apache/sysds/test/functions/builtin/BuiltinResidencyMatchTest.java create mode 100644 src/test/scripts/functions/builtin/residencymatch.dml diff --git a/dev/release/src/test/java/org/apache/sysds/validation/ValidateLicAndNotice.java b/dev/release/src/test/java/org/apache/sysds/validation/ValidateLicAndNotice.java index ec64db1e361..157b76214c6 100644 --- a/dev/release/src/test/java/org/apache/sysds/validation/ValidateLicAndNotice.java +++ b/dev/release/src/test/java/org/apache/sysds/validation/ValidateLicAndNotice.java @@ -60,7 +60,6 @@ public class ValidateLicAndNotice static final String[][] packageLicenses = { {"org/antlr", "ANTLR 4 Runtime (http://www.antlr.org/antlr4-runtime) org.antlr:antlr4-runtime:4.5.3"}, {"org/apache/wink/json4j","Apache Wink :: JSON4J (http://www.apache.org/wink/wink-json4j/) org.apache.wink:wink-json4j:1.4"}, - {"caffe","The proto file (src/main/proto/caffe/caffe.proto) is part of Caffe project,"}, {"org/tensorflow","The proto files (src/main/proto/tensorflow/event.proto and src/main/proto/tensorflow/summary.proto) is part of TensorFlow project,"}, {"jcuda","JCuda (jcuda.org)"}, }; diff --git a/scripts/builtin/residencyMatchMain.dml b/scripts/builtin/residencyMatchMain.dml new file mode 100644 index 00000000000..0094d413e3f --- /dev/null +++ b/scripts/builtin/residencyMatchMain.dml @@ -0,0 +1,194 @@ +#------------------------------------------------------------- +## Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +##------------------------------------------------------------- +# THIS SCRIPT COMPUTES A SOLUTION FOR THE HOSPITAL RESIDENCY MATCH PROBLEM +# +# INPUT PARAMETERS: +# -------------------------------------------------------------------------------------------- +# NAME TYPE DEFAULT MEANING +# -------------------------------------------------------------------------------------------- +# R Matrix --- Residents matrix R. +# It must be an ORDERED matrix. +# +# H Matrix --- Hospitals matrix H. +# It must be an UnORDERED matrix. +# +# C Matrix --- Capacity of Hospitals matrix C. +# It must be a [c,1] matrix with non zero values. +# i.e. the leftmost value in a row is the most preferred partner's index. +# i.e. the leftmost value in a row in P is the preference value for the acceptor with index 1 and vice-versa (higher is better). +# OUTPUT PARAMETERS: +# -------------------------------------------------------------------------------------------- +# NAME TYPE DEFAULT MEANING +# -------------------------------------------------------------------------------------------- +# ResidencyMatch Matrix --- Result Matrix +# If cell [i,j] is non-zero, it means that Resident i has matched with Hospital j. +# Further, if cell [i,j] is non-zero, it holds the preference value that led to the match. +# +# +# HospitalMatch Matrix --- Result Matrix +# If cell [i,j] is non-zero, it means that Resident i has matched with Hospital j. +# Further, if cell [i,j] is non-zero, it holds the preference value that led to the match. +# +# +# Residents.mtx: +# 2.0,1.0,3.0 +# 1.0,2.0,3.0 +# 1.0,2.0,0.0 +# +# Since it is an ORDERED matrix, this means that Resident 1 (row 1) likes acceptor 2 the most, followed by acceptor 1 and acceptor 3. +# If it was UNORDERED, this would mean that proposer 1 (row 1) likes acceptor 3 the most (since the value at [1,3] is the row max), +# followed by acceptor 1 (2.0 preference value) and acceptor 2 (1.0 preference value). +# +# Hospitals.mtx: +# 2.0,1.0,0.0 +# 0.0,1.0,2.0 +# 1.0,2.0,0.0 +# +# Since it is an UNORDERED matrix this means that Hospital 1 (row 1) likes Resident 1 the most (since the value at [1,1] is the row max). +# +# Capacity.mtx +# 1.0 +# 1.0 +# 1.0 +# +# ResidencyMatch.mtx +# 0.0,0.0,3.0 +# 1.0,0.0,0.0 +# 0.0,2.0,0.0 +# +# HospitalMatch.mtx +# 0.0,1.0,0.0 +# 0.0,0.0,2.0 +# 1.0,0.0,0.0 +# +# Resident 1 has matched with Hospital 3 (since [1,3] is non-zero) at a preference level of 3.0. +# Resident 2 has matched with Hospital 1 (since [2,1] is non-zero) at a preference level of 1.0. +# Resident 3 has matched with Hospital 2 (since [3,2] is non-zero) at a preference level of 2.0. +# -------------------------------------------------------------------------------------------- +m_residencyMatchMain = function(Matrix[Double] R, Matrix[Double] H,Matrix[Double] C ) + return (Matrix[Double] ResidencyMatch) +{ +#TODO PLEASE PAY ATTENTION +# in this step we consider that Residents Matrix is ORDERED !!!! +# in this step we consider that Hospital Matrix is UNORDERED !!!! + +print("STARTING Resident Hospital Match"); +#TODO set a finite number of maximum iterations so that the execution termiates after maximum iterations. + +print("\n") +print("STARTING RESIDENCY MATCH ALGORITHM"); +print("READING R as residents AND H as Hospitals and also C as capacity..."); + +m = nrow(R) +n = ncol (R) +Capacityrows = nrow(C) +####################################################################################################### +Capacity = matrix(0.0, rows=Capacityrows , cols=1) +Capacity = C; +max_position = colMaxs(Capacity) +####################################################################################################### +# we can consider number of choices for every resident # +####################################################################################################### +ResidencyMatch = matrix(0.0, rows=m, cols=n) +HospitalMatch = matrix(0.0, rows=n, cols=m) +Result_matrix = matrix(0.0, rows=nrow(R), cols=ncol(R)) +######################################################################################################## +if(nrow(Capacity) != nrow(H)) { + print("ERROR: Wrong Input !Capacity indexes is not match with the Number of hospitals ") + #it means that some hospitals' capacity is not defined. +}# end of if + +StarM = matrix(1.0, rows=m, cols=1) ### for checking while + +HIndex =matrix(1.0, rows=m, cols=1) +proposer_pointers = matrix(1.0, rows=m, cols=1) +prev_Residents_vector = matrix(1.0, rows=n, cols=1) +prevIndex_Residents_vector = matrix(1.0, rows=n, cols=1) + +prev_Residents_vector = rowMins(HospitalMatch) +prevIndex_Residents_vector = rowIndexMin(HospitalMatch) + +while(sum(StarM) > 0) { + + for(i in 1:m) { + + while (as.scalar (StarM[i]) == 1) { + SecondIndex = as.scalar (proposer_pointers[i]) + HIndex[i] = as.scalar (R[i,SecondIndex]) + prev_Residents_vector = rowMaxs(HospitalMatch) # we consider if the preference value is 1 it means it is the first selection of that hospital, the minimum value means most preference. + prevIndex_Residents_vector = rowIndexMax(HospitalMatch) + if (as.scalar(HIndex[i]) != 0 ){ + + HosValue = as.scalar (H[as.scalar (HIndex[i]),i]) + if (HosValue > 0){ + ##################### if this hospital likes this resident and has the capacity ... + if (as.scalar (Capacity[as.scalar (HIndex[i]),1]) >= 1){ + Capacity[as.scalar (HIndex[i]),1] = as.scalar (Capacity[as.scalar (HIndex[i]),1]) -1 + ResidencyMatch [i,as.scalar (HIndex[i])] = as.scalar (proposer_pointers[i]) + HospitalMatch [as.scalar (HIndex[i]), i] = HosValue + #Disable freshly Matched resident to search for a new Hospital in the next round + StarM[i] =0 + proposer_pointers[i] = as.scalar(proposer_pointers[i]) + 1 + if (as.scalar(proposer_pointers[i]) > n){ + proposer_pointers[i] = n + } + + }# #########end of if (as.scalar(HIndex[i]) != 0 )########################## + + else if (as.scalar (Capacity[as.scalar (HIndex[i]),1]) < 1) { + + if ((as.scalar (prev_Residents_vector[as.scalar (HIndex[i])]))>= SecondIndex ){ + + ##### in this step we check that if the hospital capacity is 0 but the preference value of prev residents is lower than the preference value of current resident ####### + ##### we should replace the prev resident with current resident----------------------------------------------------------------------------------------------------####### + HXX= as.scalar(prevIndex_Residents_vector[as.scalar (HIndex[i]),1]) #############which prev resident should be removed + HospitalMatch [as.scalar (HIndex[i]) ,HXX] = 0 + ResidencyMatch[HXX,as.scalar (HIndex[i])] = 0 + HospitalMatch [as.scalar (HIndex[i]),i ] = as.scalar (proposer_pointers[i]) + ResidencyMatch [i,as.scalar (HIndex[i])] = as.scalar (proposer_pointers[i]) + StarM[i] =0 + prevResIndex =as.scalar(prevIndex_Residents_vector[as.scalar (HIndex[i]),1]) + if(prevResIndex > 0){ + StarM[prevResIndex ] =1 + proposer_pointers[i] = as.scalar(proposer_pointers[i]) + 1 + if (as.scalar(proposer_pointers[i]) > n){ + proposer_pointers[i] = n + } + }# end of if(prevResIndex > 0) + }# end of checking secondIndex + }## end of else if + } #end of if X + + if ( as.scalar (StarM[i]) == 1 ){ + proposer_pointers[i] = as.scalar(proposer_pointers[i]) + 1 + if (as.scalar(proposer_pointers[i]) > n){ + proposer_pointers[i] = n + } + }###end else if hosvalue + }# end of if if (as.scalar(HIndex[i]) + # new_best_proposer_index = i + } + } ## end of for (i 1:m) +} # end of while + +print("ResidencyMatch") +print(toString(ResidencyMatch)) +print("HospitalMatch") +print(toString(HospitalMatch)) +} \ No newline at end of file diff --git a/src/assembly/extra/LICENSE b/src/assembly/extra/LICENSE index 1499ebad63d..5b8c3baca29 100644 --- a/src/assembly/extra/LICENSE +++ b/src/assembly/extra/LICENSE @@ -202,57 +202,6 @@ =============================================================================== -The proto file (src/main/proto/caffe/caffe.proto) is part of Caffe project, -which is used to generate caffe java package. -Caffe are distributed under the below license. - -COPYRIGHT - -All contributions by the University of California: -Copyright (c) 2014-2017 The Regents of the University of California (Regents) -All rights reserved. - -All other contributions: -Copyright (c) 2014-2017, the respective contributors -All rights reserved. - -Caffe uses a shared copyright model: each contributor holds copyright over -their contributions to Caffe. The project versioning records all such -contribution and copyright details. If a contributor wants to further mark -their specific copyright on a particular contribution, they should indicate -their copyright solely in the commit message of the change when it is -committed. - -LICENSE - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -CONTRIBUTION AGREEMENT - -By contributing to the BVLC/caffe repository through pull-request, comment, -or otherwise, the contributor releases their content to the -license and copyright terms herein. - -=============================================================================== - The following compile-scope dependencies come under the MIT License JCuda (jcuda.org) diff --git a/src/main/java/org/apache/sysds/common/Builtins.java b/src/main/java/org/apache/sysds/common/Builtins.java index 8c90ab9efd3..18aa5316aff 100644 --- a/src/main/java/org/apache/sysds/common/Builtins.java +++ b/src/main/java/org/apache/sysds/common/Builtins.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.sysds.common; import java.util.EnumSet; @@ -32,21 +31,25 @@ * always, user-defined DML-bodied functions take precedence over all * builtin functions. * - * To add a new builtin script function, simply add the definition here + * * To add a new builtin script function, simply add the definition here * as well as a dml file in scripts/builtin with a matching name. On * building SystemDS, these scripts are packaged into the jar as well. */ + public enum Builtins { //builtin functions - ABSTAIN("abstain", true), + STABLEMARRIAGE("stablemarriageA", true), + RESIDENCYMATCH("residencyMatchMain", true), + ARIMA("arima", true), ABS("abs", false), + GET_ACCURACY("getAccuracy", true), + ABSTAIN("abstain", true), ACOS("acos", false), ALS("als", true), ALS_CG("alsCG", true), ALS_DS("alsDS", true), ALS_PREDICT("alsPredict", true), ALS_TOPK_PREDICT("alsTopkPredict", true), - ARIMA("arima", true), ASIN("asin", false), ATAN("atan", false), AUTOENCODER2LAYER("autoencoder_2layer", true), @@ -54,9 +57,9 @@ public enum Builtins { AVG_POOL_BACKWARD("avg_pool_backward", false), BATCH_NORM2D("batch_norm2d", false, ReturnType.MULTI_RETURN), BATCH_NORM2D_BACKWARD("batch_norm2d_backward", false, ReturnType.MULTI_RETURN), + BANDIT("bandit", true), BIASADD("bias_add", false), BIASMULT("bias_multiply", false), - BANDIT("bandit", true), BITWAND("bitwAnd", false), BITWOR("bitwOr", false), BITWXOR("bitwXor", false), @@ -71,7 +74,6 @@ public enum Builtins { CAST_AS_BOOLEAN("as.logical", "as.boolean", false), CBIND("cbind", "append", false), CEIL("ceil", "ceiling", false), - CHOLESKY("cholesky", false), COLMAX("colMaxs", false), COLMEAN("colMeans", false), COLMIN("colMins", false), @@ -82,29 +84,27 @@ public enum Builtins { COLVAR("colVars", false), COMPONENTS("components", true), COMPRESS("compress", false), - CONFUSIONMATRIX("confusionMatrix", true), + CSPLINE("cspline", true), + CSPLINE_CG("csplineCG", true), + CSPLINE_DS("csplineDS", true), + DECOMPRESS("decompress", false), CONV2D("conv2d", false), CONV2D_BACKWARD_FILTER("conv2d_backward_filter", false), CONV2D_BACKWARD_DATA("conv2d_backward_data", false), - COR("cor", true), CORRECTTYPOS("correctTypos", true), COS("cos", false), - COSH("cosh", false), - COUNT_DISTINCT("countDistinct",false), - COUNT_DISTINCT_APPROX("countDistinctApprox",false), COV("cov", false), - COX("cox", true), - CSPLINE("cspline", true), - CSPLINE_CG("csplineCG", true), - CSPLINE_DS("csplineDS", true), + COSH("cosh", false), + CHOLESKY("cholesky", false), CUMMAX("cummax", false), CUMMIN("cummin", false), CUMPROD("cumprod", false), CUMSUM("cumsum", false), CUMSUMPROD("cumsumprod", false), + CONFUSIONMATRIX("confusionMatrix", true), + COR("cor", true), + COX("cox", true), DBSCAN("dbscan", true), - DECISIONTREE("decisionTree", true), - DECOMPRESS("decompress", false), DETECTSCHEMA("detectSchema", false), DENIALCONSTRAINTS("denialConstraints", true), DIAG("diag", false), @@ -123,7 +123,6 @@ public enum Builtins { FLOOR("floor", false), FRAME_SORT("frameSort", true), GAUSSIAN_CLASSIFIER("gaussianClassifier", true), - GET_ACCURACY("getAccuracy", true), GET_PERMUTATIONS("getPermutations", true), GLM("glm", true), GMM("gmm", true), @@ -134,18 +133,10 @@ public enum Builtins { IFELSE("ifelse", false), IMG_MIRROR("img_mirror", true), IMG_BRIGHTNESS("img_brightness", true), - IMG_CROP("img_crop", true), - IMG_TRANSFORM("img_transform", true), - IMG_TRANSLATE("img_translate", true), - IMG_ROTATE("img_rotate", true), - IMG_SHEAR("img_shear", true), - IMG_CUTOUT("img_cutout", true), - IMG_SAMPLE_PAIRING("img_sample_pairing", true), - IMG_INVERT("img_invert", true), - IMG_POSTERIZE("img_posterize", true), IMPUTE_BY_MEAN("imputeByMean", true), IMPUTE_BY_MEDIAN("imputeByMedian", true), IMPUTE_BY_MODE("imputeByMode", true), + IMG_CROP("img_crop", true), IMPUTE_FD("imputeByFD", true), INTERQUANTILE("interQuantile", false), INTERSECT("intersect", true), @@ -159,6 +150,7 @@ public enum Builtins { KMEANSPREDICT("kmeansPredict", true), KNNBF("knnbf", true), KNN("knn", true), + DECISIONTREE("decisionTree", true), L2SVM("l2svm", true), L2SVMPREDICT("l2svmPredict", true), LASSO("lasso", true), @@ -175,29 +167,29 @@ public enum Builtins { LSTM_BACKWARD("lstm_backward", false, ReturnType.MULTI_RETURN), LU("lu", false, ReturnType.MULTI_RETURN), MAP("map", false), + MEAN("mean", "avg", false), + MICE("mice", true), + MIN("min", "pmin", false), MAX("max", "pmax", false), MAX_POOL("max_pool", false), MAX_POOL_BACKWARD("max_pool_backward", false), - MEAN("mean", "avg", false), MEDIAN("median", false), - MICE("mice", true), - MIN("min", "pmin", false), MOMENT("moment", "centralMoment", false), MSVM("msvm", true), MSVMPREDICT("msvmPredict", true), MULTILOGREG("multiLogReg", true), MULTILOGREGPREDICT("multiLogRegPredict", true), NA_LOCF("na_locf", true), - NAIVEBAYES("naiveBayes", true, false), - NAIVEBAYESPREDICT("naiveBayesPredict", true, false), NCOL("ncol", false), NORMALIZE("normalize", true), NROW("nrow", false), + NAIVEBAYES("naiveBayes", true, false), + NAIVEBAYESPREDICT("naiveBayesPredict", true, false), OUTER("outer", false), OUTLIER("outlier", true, false), //TODO parameterize opposite - OUTLIER_ARIMA("outlierByArima",true), - OUTLIER_IQR("outlierByIQR", true), OUTLIER_SD("outlierBySd", true), + OUTLIER_IQR("outlierByIQR", true), + OUTLIER_ARIMA("outlierByArima",true), PCA("pca", true), PCAINVERSE("pcaInverse", true), PCATRANSFORM("pcaTransform", true), @@ -215,9 +207,9 @@ public enum Builtins { ROUND("round", false), ROWINDEXMAX("rowIndexMax", false), ROWINDEXMIN("rowIndexMin", false), + ROWMIN("rowMins", false), ROWMAX("rowMaxs", false), ROWMEAN("rowMeans", false), - ROWMIN("rowMins", false), ROWPROD("rowProds", false), ROWSD("rowSds", false), ROWSUM("rowSums", false), @@ -231,34 +223,34 @@ public enum Builtins { SIGN("sign", false), SIN("sin", false), SINH("sinh", false), + STEPLM("steplm",true, ReturnType.MULTI_RETURN), SLICEFINDER("slicefinder", true), SMOTE("smote", true), SOLVE("solve", false), SPLIT("split", true), SPLIT_BALANCED("splitBalanced", true), - STABLE_MARRIAGE("stableMarriage", true), STATSNA("statsNA", true), - STEPLM("steplm",true, ReturnType.MULTI_RETURN), SQRT("sqrt", false), SUM("sum", false), SVD("svd", false, ReturnType.MULTI_RETURN), + TRANS("t", false), TABLE("table", "ctable", false), TAN("tan", false), TANH("tanh", false), + TRACE("trace", false), TO_ONE_HOT("toOneHot", true), TOMEKLINK("tomeklink", true), - TRACE("trace", false), - TRANS("t", false), TYPEOF("typeof", false), - UNIVAR("univar", true), + COUNT_DISTINCT("countDistinct",false), + COUNT_DISTINCT_APPROX("countDistinctApprox",false), VAR("var", false), VECTOR_TO_CSV("vectorToCsv", true), - WINSORIZE("winsorize", true, false), //TODO parameterize w/ prob, min/max val XOR("xor", false), - + UNIVAR("univar", true), + WINSORIZE("winsorize", true, false), //TODO parameterize w/ prob, min/max val + //parameterized builtin functions CDF("cdf", false, true), - CVLM("cvlm", true, false), GROUPEDAGG("aggregate", "groupedAggregate", false, true), INVCDF("icdf", false, true), LISTNV("list", false, true), //note: builtin and parbuiltin @@ -271,15 +263,16 @@ public enum Builtins { PNORM("pnorm", false, true), PT("pt", false, true), QCHISQ("qchisq", false, true), - QEXP("qexp", false, true), QF("qf", false, true), QNORM("qnorm", false, true), QT("qt", false, true), + QEXP("qexp", false, true), REPLACE("replace", false, true), RMEMPTY("removeEmpty", false, true), SCALE("scale", true, false), SCALEAPPLY("scaleApply", true, false), TIME("time", false), + CVLM("cvlm", true, false), TOKENIZE("tokenize", false, true), TOSTRING("toString", false, true), TRANSFORMAPPLY("transformapply", false, true), @@ -388,4 +381,4 @@ public static String getInternalFName(String name, DataType dt) { return !contains(name, true, false) ? name : // private builtin (dt.isMatrix() ? "m_" : "s_") + name; // public builtin } -} +} \ No newline at end of file diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinResidencyMatchTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinResidencyMatchTest.java new file mode 100644 index 00000000000..a6a1a4a464d --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinResidencyMatchTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.builtin; + +import org.apache.sysds.runtime.matrix.data.MatrixValue; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +public class BuiltinResidencyMatchTest extends AutomatedTestBase { + + + private final static String TEST_NAME = "residencymatch"; + private final static String TEST_DIR = "functions/builtin/"; + private static final String TEST_CLASS_DIR = TEST_DIR + BuiltinResidencyMatchTest.class.getSimpleName() + "/"; + + private final static double eps = 0.0001; + private final static int rows = 3; + private final static int cols = 3; + + @Override + public void setUp() { + addTestConfiguration(TEST_NAME,new TestConfiguration(TEST_CLASS_DIR, TEST_NAME,new String[]{"SM"})); + } + @Test + public void testResidencyMatchTest() { + runtestResidencyMatchTest("residencymatch"); + } + + private void runtestResidencyMatchTest(String alg) { + loadTestConfiguration(getTestConfiguration(TEST_NAME)); + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + List proArgs = new ArrayList<>(); + proArgs.add("-args"); + proArgs.add(input("R")); + proArgs.add(input("H")); + proArgs.add(input("C")); + proArgs.add(alg); + proArgs.add(output("RM")); + + programArgs = proArgs.toArray(new String[proArgs.size()]); + // defining Residents Matrix + double[][] R = { + {2,1,3},{1,2,3},{1,3,2}}; + //defining Hospital Matrix + double[][] H = { + {3,1,2},{2,1,3},{3,2,1}}; + //defining Capacity for hospitals + double[][] C = { + {1},{1},{1}}; + double[][]EM = { // this is an expected matrix + {0,0,3},{0,2,0},{1,0,0}}; + + + writeInputMatrixWithMTD("R", R, true); + writeInputMatrixWithMTD("H", H, true); + writeInputMatrixWithMTD("C", C, true); + + + runTest(true, EXCEPTION_NOT_EXPECTED, null, -1); + + //compare expected results + HashMap matrixU = readDMLMatrixFromOutputDir("RM"); + double[][] OUT = TestUtils.convertHashMapToDoubleArray(matrixU); + System.out.println("OUTPUT"); + System.out.println( Arrays.deepToString(OUT)); + System.out.println("EXPECTED"); + System.out.println(Arrays.deepToString(EM)); + TestUtils.compareMatrices(EM, OUT, rows, cols, eps); + } +} diff --git a/src/test/scripts/functions/builtin/residencymatch.dml b/src/test/scripts/functions/builtin/residencymatch.dml new file mode 100644 index 00000000000..dd12e7ba081 --- /dev/null +++ b/src/test/scripts/functions/builtin/residencymatch.dml @@ -0,0 +1,10 @@ +# this dml file will run the ResidencyMatchMain function in builtin functions in scripts/builtin +R = read($1) +H = read($2) +C = read($3) +alg = $4 +if( alg == "residencymatch" ) +[Out] = residencyMatchMain(R=R,H=H,C=C) +write(Out, $5) + + From 9cc6916bfda5d1e988dcd3b22d9585d0020b6ad1 Mon Sep 17 00:00:00 2001 From: Atefeh Asayesh <83871787+atefeh-asayesh@users.noreply.github.com> Date: Wed, 9 Jun 2021 00:37:12 +0200 Subject: [PATCH 2/3] Delete ValidateLicAndNotice.java --- .../validation/ValidateLicAndNotice.java | 813 ------------------ 1 file changed, 813 deletions(-) delete mode 100644 dev/release/src/test/java/org/apache/sysds/validation/ValidateLicAndNotice.java diff --git a/dev/release/src/test/java/org/apache/sysds/validation/ValidateLicAndNotice.java b/dev/release/src/test/java/org/apache/sysds/validation/ValidateLicAndNotice.java deleted file mode 100644 index 157b76214c6..00000000000 --- a/dev/release/src/test/java/org/apache/sysds/validation/ValidateLicAndNotice.java +++ /dev/null @@ -1,813 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysds.validation; - -import java.io.BufferedReader; -import java.io.BufferedOutputStream; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.IOException; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.StringTokenizer; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; - -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; -import org.apache.commons.io.FileUtils; - - -/** - * Checks that all jar files included in the distribution are mentioned in the LICENSE file - * and that all jars mentioned in the LICENSE are in the distribution. - */ -public class ValidateLicAndNotice -{ - public static String[] fileTypes = {Constants.JAR, Constants.DLL, Constants.EXP, Constants.LIB, Constants.PDB, Constants.EXE}; - - // Zip Distribution directory. - private String strDistroDir = "../../../target/release/systemds/target/"; - - static final String[][] packageLicenses = - { {"org/antlr", "ANTLR 4 Runtime (http://www.antlr.org/antlr4-runtime) org.antlr:antlr4-runtime:4.5.3"}, - {"org/apache/wink/json4j","Apache Wink :: JSON4J (http://www.apache.org/wink/wink-json4j/) org.apache.wink:wink-json4j:1.4"}, - {"org/tensorflow","The proto files (src/main/proto/tensorflow/event.proto and src/main/proto/tensorflow/summary.proto) is part of TensorFlow project,"}, - {"jcuda","JCuda (jcuda.org)"}, - }; - - public static HashMap hmJSLicenses = new HashMap(); - static { - - String [][] strTemp1 = {{"Bootstrap v3.3.6", "Copyright (c) 2011-2015 Twitter, Inc.", "false"}}; - hmJSLicenses.put("bootstrap.min.js", strTemp1); - String [][] strTemp2 = {{"Normalize v3.0.3", "Copyright (c) Nicolas Gallagher and Jonathan Neal", "false"}}; - hmJSLicenses.put("bootstrap.min.css", strTemp2); - String [][] strTemp3 = {{"AnchorJS v1.1.1", "Copyright (c) 2015 Bryan Braun", "false"}}; - hmJSLicenses.put("anchor.min.js", strTemp3); - String [][] strTemp4 = {{"jQuery v1.12.0", "(c) jQuery Foundation", "false"}, - {"jQuery v1.12.0", "Copyright jQuery Foundation and other contributors, https://jquery.org/", "false"}}; - hmJSLicenses.put("jquery-1.12.0.min.js", strTemp4); - String [][] strTemp5 = {{"Pygments", "Copyright (c) 2006-2017 by the respective authors (see AUTHORS file).", "false"}}; - hmJSLicenses.put("pygments-default.css", strTemp5); - } - - public ValidateLicAndNotice() { - } - - public ValidateLicAndNotice(String strDistroDir) { - setDistroDir(strDistroDir); - } - - public String getDistroDir() { - return strDistroDir; - } - - public void setDistroDir(String strDistroDir) { - this.strDistroDir = strDistroDir; - } - - /** - * This will validate all archives from distribution location. - * - * @return Returns the output code - */ - public int validate() throws Exception { - - int retCode = Constants.SUCCESS, retCodeForAllFileTypes = Constants.SUCCESS, retCodeAll = Constants.SUCCESS; - - File distroRoot = new File( getDistroDir()); - File libDirectory = distroRoot; - if (!libDirectory.exists()) { - Utility.debugPrint(Constants.DEBUG_ERROR, "Distribution folder '" + libDirectory.getAbsoluteFile().toString() + "' does not exist."); - return Constants.NO_ZIP_TGZ; - } - - File outTempDir = File.createTempFile("outTemp", ""); - outTempDir.delete(); - outTempDir.mkdir(); - - List zips = getZipsInDistro(libDirectory); - if(zips.size() == 0) { - Utility.debugPrint(Constants.DEBUG_ERROR, "Can't find archives in folder: " + libDirectory.getAbsoluteFile().toString()); - return Constants.NO_ZIP_TGZ; - } - - for (String zipFile: zips) - { - retCodeForAllFileTypes = Constants.SUCCESS; - Utility.debugPrint(Constants.DEBUG_INFO, "======================================================================================"); - Utility.debugPrint(Constants.DEBUG_INFO, "Validating archive: " + zipFile + " ..."); - - for (String fileType: fileTypes) { - retCode = Constants.SUCCESS; - - List filesAll = null; - // Extract license/notice only at first time in all filetypes validation for a given zip. - if(fileType == Constants.JAR) { - if (!ValidateLicAndNotice.extractFile(libDirectory + "/" + zipFile, Constants.LICENSE, outTempDir.getAbsolutePath(), true)) - return Constants.FAILED_TO_EXTRACT; - if (!ValidateLicAndNotice.extractFile(libDirectory + "/" + zipFile, Constants.NOTICE, outTempDir.getAbsolutePath(), true)) - return Constants.FAILED_TO_EXTRACT; - } - - filesAll = getFiles(libDirectory + "/" + zipFile, fileType); - - File licenseFile = new File(outTempDir, Constants.LICENSE); - List files = new ArrayList(); - List fileSysds = new ArrayList(); - for (String file : filesAll) { - int sysdsLen = Constants.SYSTEMDS_NAME.length(); - String strBegPart = file.substring(0, sysdsLen); - if (strBegPart.compareToIgnoreCase(Constants.SYSTEMDS_NAME) != 0) - files.add(file); - else - fileSysds.add(file); - } - - - List bad2 = getLICENSEFilesNotInList(licenseFile, files, fileType); - if (bad2.size() > 0) { - Utility.debugPrint(Constants.DEBUG_WARNING,"Files in LICENSE but not in Distribution: " + bad2); - retCode += Constants.FILE_NOT_IN_ZIP; - } - - List bad1 = getFilesNotInLICENSE(licenseFile, files, fileType); - if (bad1.size() > 0) { - Utility.debugPrint(Constants.DEBUG_ERROR,"Files in distribution but not in LICENSE: " + bad1); - retCode += Constants.FILE_NOT_IN_LIC; - } - - // Validate shaded jar and notice only one time for each archive. - if(fileType == Constants.JAR) { - for (String file : fileSysds) - retCode += ValidateLicAndNotice.validateShadedLic(libDirectory + "/" + zipFile, file, outTempDir.getAbsolutePath()); - if (!validateNotice(outTempDir.getAbsolutePath()+"/"+Constants.NOTICE)) { - Utility.debugPrint(Constants.DEBUG_ERROR, "Notice validation failed, please check notice file manually in this archive."); - retCode += Constants.INVALID_NOTICE; - } - if (!validateJSCssLicense(licenseFile, libDirectory + "/" + zipFile)) { - Utility.debugPrint(Constants.DEBUG_ERROR, "JS/CSS license validation failed, please check license file manually in this archive."); - retCode += Constants.JS_CSS_LIC_NOT_EXIST; - } - } - - if (retCode == Constants.SUCCESS) - Utility.debugPrint(Constants.DEBUG_INFO3, "Validation of file type '." + fileType + "' in archive " + zipFile + " completed successfully."); - else { - Utility.debugPrint(Constants.DEBUG_ERROR, "License/Notice validation failed for archive " + zipFile + " with error code " + retCode + ", please validate file manually."); - retCodeForAllFileTypes = Constants.FAILURE; - } - } - if(retCodeForAllFileTypes == Constants.SUCCESS) - Utility.debugPrint(Constants.DEBUG_INFO, "Validation of archive " + zipFile + " completed successfully."); - - retCodeAll = retCodeForAllFileTypes != Constants.SUCCESS?Constants.FAILURE:retCodeAll; - } - Utility.debugPrint(Constants.DEBUG_INFO, "======================================================================================"); - - FileUtils.deleteDirectory(outTempDir); - return retCodeAll; - } - - /** - * This will validate objects (class files) from jar file within a zip file. - * - * @param zipFileName is the name of zip file from which set of class packages will be returned. - * @param fileName is the name of the file within zip (jar) file to validate list of packages within. - * @param outTempDir is the temporary directory name. - * @return Success or Failure code - */ - public static int validateShadedLic(String zipFileName, String file, String outTempDir) throws Exception - { - - File outTempDir2 = new File (outTempDir + "/" + "2"); - outTempDir2.mkdir(); - if(!ValidateLicAndNotice.extractFile(zipFileName, file, outTempDir2.getAbsolutePath(), false)) - return Constants.FAILED_TO_EXTRACT; - if(!ValidateLicAndNotice.extractFile(outTempDir2.getAbsolutePath()+"/"+file, Constants.LICENSE, outTempDir2.getAbsolutePath(), true)) - return Constants.FAILED_TO_EXTRACT; - - HashMap hashMapPackages = getPackagesFromZip(outTempDir2.getAbsolutePath() + "/" + file); - for (String packageName: hashMapPackages.keySet()) - Utility.debugPrint(Constants.DEBUG_CODE, "Package: " + packageName + " Licensed: " + hashMapPackages.get(packageName)); - - int iRetCode = ValidateLicAndNotice.validatePackages(outTempDir2.getAbsolutePath()+"/"+Constants.LICENSE, hashMapPackages); - - FileUtils.deleteDirectory(outTempDir2); - return iRetCode; - } - - /** - * This will validate objects (class files) against license file. - * - * @param licenseFile is the name of the license file. - * @param hashMapPackages is the list of package names to be validated for license. - * @return Success or Failure code - */ - public static int validatePackages(String licenseFile, HashMap hashMapPackages) throws Exception - { - int iRetCode = Constants.SUCCESS; - BufferedReader reader = new BufferedReader(new FileReader(licenseFile)); - String line = null; - HashSet packageValidLic = new HashSet(); - while ((line = reader.readLine()) != null) { - line = line.trim(); - for(int i=0; i > itPackages = hashMapPackages.entrySet().iterator(); - while (itPackages.hasNext()) { - Map.Entry pairPackage = (Map.Entry) itPackages.next(); - Iterator itLicPackages = packageValidLic.iterator(); - while (itLicPackages.hasNext()) { - if(((String)pairPackage.getKey()).startsWith((String)itLicPackages.next())) - pairPackage.setValue(Boolean.TRUE); - } - } - - itPackages = hashMapPackages.entrySet().iterator(); - while (itPackages.hasNext()) { - Map.Entry pairPackage = (Map.Entry) itPackages.next(); - if(!(Boolean)pairPackage.getValue()) { - Utility.debugPrint(Constants.DEBUG_WARNING, "Could not validate license for package " + pairPackage.getKey() + ", please validate manually."); - iRetCode = Constants.LIC_NOT_EXIST; - } - } - - return iRetCode; - } - - /** - * This will return the set of packages from zip file. - * - * @param zipFileName is the name of zip file from which set of class packages will be returned. - * @return Returns set of packages for classes included in the zip file . - */ - public static HashMap getPackagesFromZip (String zipFileName) throws Exception{ - HashMap packages = new HashMap(); - try { - ZipEntry entry; - ZipFile zipfile = new ZipFile(zipFileName); - Enumeration e = zipfile.entries(); - while(e.hasMoreElements()) { - entry = (ZipEntry) e.nextElement(); - if(! entry.getName().startsWith(Constants.SYSTEMDS_PACKAGE) && - entry.getName().endsWith("." + Constants.CLASS)) { - int iPos = entry.getName().lastIndexOf("/"); - if (iPos > 0) { - String strPackageName = entry.getName().substring(0, iPos); - packages.put(strPackageName, Boolean.FALSE); - Utility.debugPrint(Constants.DEBUG_CODE, "Package found : " + strPackageName); - } - } - } - } catch(Exception e) { - e.printStackTrace(); - throw e; - } - return packages; - } - - /** - * This will return the list of files in license files but not in list of files coming from archive. - * - * @param licenseFile is the file against which contents of archive gets compared. - * @param files are the list of files coming from archive. - * @param fileExt is the extension of file to validate (e.g. "jar") - * @return Returns the list of files in License file but not in archive. - */ - private List getLICENSEFilesNotInList(File licenseFile, List files, String fileExt) throws IOException { - - List badFiles = new ArrayList(); - BufferedReader reader = new BufferedReader(new FileReader(licenseFile)); - String line = null; - while ((line = reader.readLine()) != null) { - line = line.trim(); - if (line.contains("." + fileExt)) { - StringTokenizer st = new StringTokenizer(line); - while (st.hasMoreTokens()) { - String s = st.nextToken(); - if (s.contains("." + fileExt)) { - if (s.startsWith("(")) { - s = s.substring(1); - } - if (s.endsWith(",") || s.endsWith(":")) { - s = s.substring(0, s.length()-1); - } - if (s.endsWith(")")) { - s = s.substring(0, s.length()-1); - } - if (!files.contains(s)) { - badFiles.add(s); - } - } - } - } - } - return badFiles; - } - - /** - * This will return the list of files in license files with specified file extension. - * - * @param licenseFile is the file against which contents of archive gets compared. - * @param fileExt is the extension of file to validate (e.g. "jar") - * @return Returns the list of files in License file. - */ - private List getFilesFromLicenseFile(File licenseFile, String fileExt) throws IOException { - - List files = new ArrayList(); - BufferedReader reader = new BufferedReader(new FileReader(licenseFile)); - String line = null; - while ((line = reader.readLine()) != null) { - line = line.trim(); - if (line.contains("." + fileExt)) { - StringTokenizer st = new StringTokenizer(line); - while (st.hasMoreTokens()) { - String s = st.nextToken(); - if (s.contains("." + fileExt)) { - if (s.startsWith("(")) { - s = s.substring(1); - } - if (s.endsWith(",") || s.endsWith(":")) { - s = s.substring(0, s.length()-1); - } - if (s.endsWith(")")) { - s = s.substring(0, s.length()-1); - } - if (!files.contains(s)) { - files.add(s); - } - } - } - } - } - return files; - } - - /** - * This will return the list of files coming from archive but not in the license file. - * - * @param licenseFile is the file against which contents of archive gets compared. - * @param files are the list of files coming from archive. - * @param fileExt is the extension of file to validate (e.g. "jar") - * @return Returns the list of files in archive but not in License file. - */ - private List getFilesNotInLICENSE(File licenseFile, List files, String fileExt) throws IOException { - List badFiles = new ArrayList(); - List licFiles = getFilesFromLicenseFile(licenseFile, fileExt); - for (String file : files) { - if (!licFiles.contains(file)) { - badFiles.add(file); - } - } - return badFiles; - } - - /** - * This will return the list of archives from a directory. - * - * @param directory is the location from where list of archives will be returned. - * @return Returns the list of archives (e.g., .zip/tgz/tar.gz files) from a directory. - */ - private List getZipsInDistro(File directory) { - List zips = new ArrayList(); - for (String fileName : directory.list()) - if ((fileName.endsWith("." + Constants.ZIP)) || (fileName.endsWith("." + Constants.TGZ)) || - (fileName.endsWith("." + Constants.TAR_GZ))) { - zips.add(fileName); - } - return zips; - } - - /** - * This will return the content of file. - * - * @param file is the parameter of type File from which contents will be read and returned. - * @return Returns the contents from file in String format. - */ - private static String readFile(File file) throws java.io.IOException { - StringBuffer fileData = new StringBuffer(); - BufferedReader reader = new BufferedReader(new FileReader(file)); - char[] buf = new char[1024]; - int numRead = 0; - while ((numRead = reader.read(buf)) != -1) { - String readData = String.valueOf(buf, 0, numRead); - fileData.append(readData); - buf = new char[1024]; - } - reader.close(); - return fileData.toString(); - } - - /** - * This will return the file from archive and store it in specified location. - * - * @param zipFileName is the name of archive from which file to be extracted. - * @param fileName is the name of the file to be extracted. - * @param strDestLoc is the location where file will be extracted. - * @param bFirstDirLevel to indicate to get file from first directory level. - * @return Success or Failure - */ - public static boolean extractFile(String zipFileName, String fileName, String strDestLoc, boolean bFirstDirLevel) { - Utility.debugPrint(Constants.DEBUG_CODE, "Extracting " + fileName + " from archive " + zipFileName); - if (zipFileName.endsWith("." + Constants.ZIP) || zipFileName.endsWith("." + Constants.JAR)) - return extractFileFromZip(zipFileName, fileName, strDestLoc, bFirstDirLevel); - else if (zipFileName.endsWith("." + Constants.TGZ) || zipFileName.endsWith("." + Constants.TAR_GZ)) - return extractFileFromTGZ(zipFileName, fileName, strDestLoc, bFirstDirLevel); - return Constants.bFAILURE; - } - - - /** - * This will return the file from zip file and store it in specified location. - * - * @param zipFileName is the name of zip file from which file to be extracted. - * @param fileName is the name of the file to be extracted. - * @param strDestLoc is the location where file will be extracted. - * @param bFirstDirLevel to indicate to get file from first directory level. - * @return Sucess or Failure - */ - public static boolean extractFileFromZip (String zipFileName, String fileName, String strDestLoc, boolean bFirstDirLevel) { - boolean bRetCode = Constants.bFAILURE; - try { - BufferedOutputStream bufOut = null; - BufferedInputStream bufIn = null; - ZipEntry entry; - ZipFile zipfile = new ZipFile(zipFileName); - Enumeration e = zipfile.entries(); - while(e.hasMoreElements()) { - entry = (ZipEntry) e.nextElement(); - if(! entry.getName().endsWith(fileName)) - continue; - //Get file at root (in single directory) level. This is for License in root location. - if( bFirstDirLevel && - (entry.getName().indexOf('/') != entry.getName().lastIndexOf('/'))) - continue; - bufIn = new BufferedInputStream(zipfile.getInputStream(entry)); - int count; - byte data[] = new byte[Constants.BUFFER]; - String strOutFileName = strDestLoc == null ? entry.getName(): strDestLoc + "/" + fileName; - FileOutputStream fos = new FileOutputStream(strOutFileName); - bufOut = new BufferedOutputStream(fos, Constants.BUFFER); - while ((count = bufIn.read(data, 0, Constants.BUFFER)) != -1) { - bufOut.write(data, 0, count); - } - bufOut.flush(); - bufOut.close(); - bufIn.close(); - bRetCode = Constants.bSUCCESS; - break; - } - } catch(Exception e) { - e.printStackTrace(); - } - return bRetCode; - } - - /** - * This will return the file from tgz file and store it in specified location. - * - * @param tgzFileName is the name of tgz file from which file to be extracted. - * @param fileName is the name of the file to be extracted. - * @param strDestLoc is the location where file will be extracted. - * @param bFirstDirLevel to indicate to get file from first directory level. - * @return Sucess or Failure - */ - public static boolean extractFileFromTGZ (String tgzFileName, String fileName, String strDestLoc, boolean bFirstDirLevel) { - - boolean bRetCode = Constants.bFAILURE; - - TarArchiveInputStream tarIn = null; - - try { - - tarIn = new TarArchiveInputStream( - new GzipCompressorInputStream( - new BufferedInputStream( - new FileInputStream(tgzFileName)))); - } catch(Exception e) { - Utility.debugPrint(Constants.DEBUG_ERROR, "Exception in unzipping tar file: " + e); - return bRetCode; - } - - try { - BufferedOutputStream bufOut = null; - BufferedInputStream bufIn = null; - TarArchiveEntry tarEntry = null; - while((tarEntry = tarIn.getNextTarEntry()) != null) { - if(! tarEntry.getName().endsWith(fileName)) - continue; - //Get file at root (in single directory) level. This is for License in root location. - if( bFirstDirLevel && - (tarEntry.getName().indexOf('/') != tarEntry.getName().lastIndexOf('/'))) - continue; - bufIn = new BufferedInputStream (tarIn); - int count; - byte data[] = new byte[Constants.BUFFER]; - String strOutFileName = strDestLoc == null ? tarEntry.getName(): strDestLoc + "/" + fileName; - FileOutputStream fos = new FileOutputStream(strOutFileName); - bufOut = new BufferedOutputStream(fos, Constants.BUFFER); - while ((count = bufIn.read(data, 0, Constants.BUFFER)) != -1) { - bufOut.write(data, 0, count); - } - bufOut.flush(); - bufOut.close(); - bufIn.close(); - bRetCode = Constants.bSUCCESS; - break; - } - } catch(Exception e) { - e.printStackTrace(); - } - return bRetCode; - } - - /** - * This will return the list of files from archive. - * - * @param zipFileName is the name of archive (e.g., .zip/tgz/tar.gz file) from which list of files with specified file extension will be returned. - * @param fileExt is the file extension to be used to get list of files to be returned. - * @return Returns list of files having specified extension from archive. - */ - public static List getFiles (String zipFileName, String fileExt) { - if (zipFileName.endsWith("." + Constants.ZIP)) - return getFilesFromZip (zipFileName, fileExt); - else if (zipFileName.endsWith("." + Constants.TGZ) || zipFileName.endsWith("." + Constants.TAR_GZ)) - return getFilesFromTGZ (zipFileName, fileExt); - return null; - } - /** - * This will return the list of files from zip file. - * - * @param zipFileName is the name of zip file from which list of files with specified file extension will be returned. - * @param fileExt is the file extension to be used to get list of files to be returned. - * @return Returns list of files having specified extension from zip file. - */ - public static List getFilesFromZip (String zipFileName, String fileExt) { - List files = new ArrayList(); - try { - ZipEntry entry; - ZipFile zipfile = new ZipFile(zipFileName); - Enumeration e = zipfile.entries(); - while(e.hasMoreElements()) { - entry = (ZipEntry) e.nextElement(); - if(entry.getName().endsWith("." + fileExt)) { - int iPos = entry.getName().lastIndexOf("/"); - if (iPos == 0) - --iPos; - String strFileName = entry.getName().substring(iPos+1); - files.add(strFileName); - } - } - } catch(Exception e) { - e.printStackTrace(); - } - return (files); - } - - /** - * This will return the list of files from tgz file. - * - * @param tgzFileName is the name of tgz file from which list of files with specified file extension will be returned. - * @param fileExt is the file extension to be used to get list of files to be returned. - * @return Returns list of files having specified extension from tgz file. - */ - public static List getFilesFromTGZ (String tgzFileName, String fileExt) { - - TarArchiveInputStream tarIn = null; - - try { - - tarIn = new TarArchiveInputStream( - new GzipCompressorInputStream( - new BufferedInputStream( - new FileInputStream(tgzFileName)))); - } catch(Exception e) { - Utility.debugPrint(Constants.DEBUG_ERROR, "Exception in unzipping tar file: " + e); - return null; - } - - List files = new ArrayList(); - try { - TarArchiveEntry tarEntry = null; - while((tarEntry = tarIn.getNextTarEntry()) != null) { - if(tarEntry.getName().endsWith("." + fileExt)) { - int iPos = tarEntry.getName().lastIndexOf("/"); - if (iPos == 0) - --iPos; - String strFileName = tarEntry.getName().substring(iPos+1); - files.add(strFileName); - } - } - } catch(Exception e) { - e.printStackTrace(); - } - return (files); - } - - /** - * This will return if NOTICE file is valid or not. - * - * @param noticeFile is the noticew file to be verified. - * @return Returns if NOTICE file validatation successful or failure. - */ - public static boolean validateNotice(String noticeFile) throws Exception { - - boolean bValidNotice = Constants.bSUCCESS; - - LocalDateTime currentTime = LocalDateTime.now(); - - String noticeLines[] = new String[2]; - boolean noticeLineIn[] = new boolean[4]; - -// ToDo: fix notice lines - noticeLines[0] = "Apache SystemDS"; - noticeLines[1] = "Copyright [2015-" + currentTime.getYear() + "]"; - - BufferedReader reader = new BufferedReader(new FileReader(noticeFile)); - String line = null; - while ((line = reader.readLine()) != null) { - line = line.trim(); - - for (int i = 0; i < noticeLines.length; i++) - if (line.contains(noticeLines[i])) - noticeLineIn[i] = true; - } - - for (int i = 0; i < noticeLines.length; i++) { - if (!noticeLineIn[i]) { - bValidNotice = Constants.bFAILURE; - } - } - - if(bValidNotice == Constants.bSUCCESS) - Utility.debugPrint(Constants.DEBUG_INFO2, "Notice validation successful."); - - return bValidNotice; - } - - /** - * This will validate license for JavaScript & CSS files within an archive. - * - * @param licenseFile is the file against which contents of archive gets compared. - * @param zipFileName is the name of archive from which list of JavaScript files will be returned. - * @return Success or Failure code - */ - public static boolean validateJSCssLicense(File licenseFile, String zipFileName) throws Exception - { - boolean bRetCode = Constants.bSUCCESS; - - try { - List jsFiles = getFiles(zipFileName, Constants.JS); - List cssFiles = getFiles(zipFileName, Constants.CSS); - HashMap jsCssFileHashMap = new HashMap(); - for (String jsFile : jsFiles) - if(jsFile.compareTo("main.js") != 0) - jsCssFileHashMap.put(jsFile, Boolean.FALSE); - for (String cssFile : cssFiles) - if(cssFile.compareTo("main.css") != 0) - jsCssFileHashMap.put(cssFile, Boolean.FALSE); - - BufferedReader reader = new BufferedReader(new FileReader(licenseFile)); - String line = null; - HashSet packageValidLic = new HashSet(); - while ((line = reader.readLine()) != null) { - line = line.trim(); - //Move to beginning of individual License text - if (line.startsWith(Constants.LIC_TEXT_DELIM)) - break; - } - - while ((line = reader.readLine()) != null) { - line = line.trim(); - - List curLicense = new ArrayList(); - //Read all lines until end of individual License text - while (!line.startsWith(Constants.LIC_TEXT_DELIM)) { - curLicense.add(line); - if ((line = reader.readLine()) == null) - break; - } - - //Verify jsFiles against current license foumd. - Iterator> itJSLicenses = hmJSLicenses.entrySet().iterator(); - while (itJSLicenses.hasNext()) { - Map.Entry pairJSLicense = (Map.Entry) itJSLicenses.next(); - - String[][] JSLicenseList = pairJSLicense.getValue(); - - for (String[] license : JSLicenseList) { - boolean bLicFirstPartFound = false; - - for (String licLine : curLicense) { - if (!bLicFirstPartFound && licLine.startsWith(license[0])) - bLicFirstPartFound = true; - - if (bLicFirstPartFound && licLine.contains(license[1])) { - license[2] = "true"; - break; - } - } - } - } - } - - //Validate all js/css files against license found in LICENSE file. - Iterator> itJSCssFiles = jsCssFileHashMap.entrySet().iterator(); - while (itJSCssFiles.hasNext()) { - Map.Entry pairJSCSSFile = (Map.Entry) itJSCssFiles.next(); - - String[][] jsFileLicList = hmJSLicenses.get(pairJSCSSFile.getKey()); - if(jsFileLicList == null) { - Utility.debugPrint(Constants.DEBUG_WARNING, "JS/CSS license does not exist for file " + pairJSCSSFile.getKey()); - bRetCode = Constants.bFAILURE; - continue; - } - - boolean bValidLic = true; - for (String[] jsFileLic : jsFileLicList) { - if (jsFileLic[2].compareTo("true") != 0) { - bValidLic = false; - break; - } - } - - if (bValidLic) { - jsCssFileHashMap.put(pairJSCSSFile.getKey(), Boolean.TRUE); - Utility.debugPrint(Constants.DEBUG_INFO3, "JS/CSS license exists for file " + pairJSCSSFile.getKey()); - } - else { - Utility.debugPrint(Constants.DEBUG_WARNING, "JS/CSS license does not exist for file " + pairJSCSSFile.getKey()); - bRetCode = Constants.bFAILURE; - } - } - - if (bRetCode == Constants.bSUCCESS) - Utility.debugPrint(Constants.DEBUG_INFO2, "JS/CSS license validation successful."); - - } catch (Exception e) { - System.out.println(e); - e.printStackTrace(); - } - return bRetCode; - } - - /** - * This is main() program. - * - * @param args is list of arguments - * @return - */ - public static void main(String [] args) { - ValidateLicAndNotice valLic; - if (args.length > 0) - valLic = new ValidateLicAndNotice(args[0]); - else - valLic = new ValidateLicAndNotice(); - - try { - int retCode = valLic.validate(); - - Utility.debugPrint(Constants.DEBUG_INFO, "Return code = " + retCode); - } - catch (Exception e) { - Utility.debugPrint(Constants.DEBUG_ERROR, "Error while validating license in archive: " + e); - } - } - -} From ef56ac738c313ffedbb2eadc5231d5203f5a78e9 Mon Sep 17 00:00:00 2001 From: Atefeh Asayesh <83871787+atefeh-asayesh@users.noreply.github.com> Date: Wed, 9 Jun 2021 00:37:36 +0200 Subject: [PATCH 3/3] Delete LICENSE --- src/assembly/extra/LICENSE | 249 ------------------------------------- 1 file changed, 249 deletions(-) delete mode 100644 src/assembly/extra/LICENSE diff --git a/src/assembly/extra/LICENSE b/src/assembly/extra/LICENSE deleted file mode 100644 index 5b8c3baca29..00000000000 --- a/src/assembly/extra/LICENSE +++ /dev/null @@ -1,249 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -=============================================================================== - -The following compile-scope dependencies come under the MIT License - -JCuda (jcuda.org) - -org.jcuda:jcuda:10.2.0 -org.jcuda:jcublas:10.2.0 -org.jcuda:jcufft:10.2.0 -org.jcuda:jcusparse:10.2.0 -org.jcuda:jcusolver:10.2.0 -org.jcuda:jcurand:10.2.0 -org.jcuda:jnvgraph:10.2.0 -org.jcuda:jcudnn:10.2.0 -org.jcuda:jcuda-natives:10.2.0 -org.jcuda:jcublas-natives:10.2.0 -org.jcuda:jcufft-natives:10.2.0 -org.jcuda:jcusparse-natives:10.2.0 -org.jcuda:jcusolver-natives:10.2.0 -org.jcuda:jcurand-natives:10.2.0 -org.jcuda:jnvgraph-natives:10.2.0 -org.jcuda:jcudnn-natives:10.2.0 - - -The MIT License (MIT) - -Copyright (c) 2008-2016 Marco Hutter - http://www.jcuda.org - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -===============================================================================