Merge pull request #30 from ow2-proactive/master

synchronize fork

build.gradle

@@ -7,6 +7,7 @@ buildscript {
         maven { url "https://plugins.gradle.org/m2/" }
         maven { url 'http://repository.activeeon.com/content/groups/proactive/' }
         maven { url "http://nexus.qmino.com/content/repositories/miredot" }
+        maven { url "https://plugins.gradle.org/m2/" }
     }
 
     dependencies {
@@ -30,11 +31,29 @@ buildscript {
         classpath "org.sonarsource.scanner.gradle:sonarqube-gradle-plugin:1.2"
         classpath "com.diffplug.gradle.spotless:spotless:2.4.0"
         classpath "org.ow2.proactive:coding-rules:1.0.0"
+        classpath "gradle.plugin.net.foragerr.jmeter:jmeter-gradle-plugin:1.0.7-2.13" // plugin which is used to run jmeter which launches our performance tests
+
         delete "gradle/ext"
         ant.unjar src: configurations.classpath.find { it.name.startsWith("coding-rules") }, dest: 'gradle/ext'
     }
 }
 
+logging.captureStandardOutput(LogLevel.DEBUG)
+
+//logging {
+//    file ('build.log', LogLevel.DEBUG)
+//    console (LogLevel.INFO)
+//}
+def fileLogger = [
+        onOutput : {
+            File logfile = new File( 'gradle.log' )
+            logfile << it
+        }
+] as org.gradle.api.logging.StandardOutputListener
+
+allprojects { logging.addStandardOutputListener( fileLogger ) }
+
+
 def isWindows = System.properties['os.name'].toLowerCase().contains('windows')
 
 def custom ={ "$rootDir/gradle/${it}.gradle"}
@@ -103,7 +122,6 @@ configure(javaSubprojects) {
 
     sourceCompatibility = JavaVersion.VERSION_1_7
     targetCompatibility = JavaVersion.VERSION_1_8
-
     compileJava.options.compilerArgs << '-Xlint:-options' // remove warning about bootstrap class path
 
     dependencies {
@@ -126,6 +144,7 @@ configure(javaSubprojects) {
         exclude 'functionaltests/**'
         exclude 'unittests/**'
         exclude 'unitTests/**'
+        exclude 'performancetests/**'
 
         if(Boolean.valueOf(System.getProperty("java.awt.headless"))){
             exclude 'org/ow2/proactive_grid_cloud_portal/cli/**'
@@ -360,6 +379,10 @@ subprojects {
     tasks.withType(Test).matching { it.name == 'functionalTest' }*.dependsOn dist
 }
 
+subprojects {
+    task allDeps(type: DependencyReportTask) {}
+}
+
 apply plugin: 'distribution'
 
 distributions {

common/common-api/src/main/java/org/ow2/proactive/scripting/Script.java

@@ -39,6 +39,8 @@
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
@@ -80,7 +82,7 @@ public abstract class Script<E> implements Serializable {
     public static final String MD5 = "MD5";
 
     /** Name of the script engine or file path to script file (extension will be used to lookup) */
-    protected String scriptEngineLookup;
+    protected String scriptEngineLookupName;
 
     /** The script to evaluate */
     protected String script;
@@ -105,7 +107,7 @@ public Script() {
      * @throws InvalidScriptException if the creation fails.
      */
     public Script(String script, String engineName, Serializable[] parameters) throws InvalidScriptException {
-        this.scriptEngineLookup = engineName;
+        this.scriptEngineLookupName = engineName;
         this.script = script;
         this.id = script;
         this.parameters = parameters;
@@ -123,7 +125,7 @@ public Script(String script, String engineName, Serializable[] parameters) throw
      */
     public Script(String script, String engineName, Serializable[] parameters, String scriptName)
             throws InvalidScriptException {
-        this.scriptEngineLookup = engineName;
+        this.scriptEngineLookupName = engineName;
         this.script = script;
         this.id = script;
         this.parameters = parameters;
@@ -156,7 +158,7 @@ public Script(String script, String engineName, String scriptName) throws Invali
      * @throws InvalidScriptException if the creation fails.
      */
     public Script(File file, Serializable[] parameters) throws InvalidScriptException {
-        this.scriptEngineLookup = FileUtils.getExtension(file.getPath());
+        this.scriptEngineLookupName = FileUtils.getExtension(file.getPath());
 
         try {
             script = readFile(file);
@@ -182,7 +184,7 @@ public Script(File file) throws InvalidScriptException {
      * @throws InvalidScriptException if the creation fails.
      */
     public Script(URL url, Serializable[] parameters) throws InvalidScriptException {
-        this.scriptEngineLookup = FileUtils.getExtension(url.getFile());
+        this.scriptEngineLookupName = FileUtils.getExtension(url.getFile());
 
         try {
             storeScript(url);
@@ -208,15 +210,15 @@ public Script(URL url) throws InvalidScriptException {
      * @throws InvalidScriptException if the creation fails.
      */
     public Script(Script<?> script2) throws InvalidScriptException {
-        this(script2.getScript(), script2.scriptEngineLookup, script2.getParameters(), script2.getScriptName());
+        this(script2.getScript(), script2.scriptEngineLookupName, script2.getParameters(), script2.getScriptName());
     }
 
     /** Create a script from another script object
      * @param script2 script object source
      * @throws InvalidScriptException if the creation fails.
      */
     public Script(Script<?> script2, String scriptName) throws InvalidScriptException {
-        this(script2.script, script2.scriptEngineLookup, script2.parameters, scriptName);
+        this(script2.script, script2.scriptEngineLookupName, script2.parameters, scriptName);
     }
 
     /**
@@ -279,7 +281,7 @@ public ScriptResult<E> execute(Map<String, Object> aBindings, PrintStream output
 
         if (engine == null)
             return new ScriptResult<>(new Exception("No Script Engine Found for name or extension " +
-                                                    scriptEngineLookup));
+                                                    scriptEngineLookupName));
 
         // SCHEDULING-1532: redirect script output to a buffer (keep the latest DEFAULT_OUTPUT_MAX_SIZE)
         BoundedStringWriter outputBoundedWriter = new BoundedStringWriter(outputSink, DEFAULT_OUTPUT_MAX_SIZE);
@@ -347,20 +349,54 @@ protected Reader getReader() {
 
     /** The Script Engine used to evaluate the script. */
     protected ScriptEngine createScriptEngine() {
+
+        Map<ScriptEngine, Integer> scriptEngineCandidates;
+        final boolean findByName = true;
+        scriptEngineCandidates = findScriptEngineCandidates(findByName);
+
+        if (scriptEngineCandidates.isEmpty()) {
+            scriptEngineCandidates = findScriptEngineCandidates(!findByName);
+        }
+
+        return findBestScriptEngine(scriptEngineCandidates);
+    }
+
+    private Map<ScriptEngine, Integer> findScriptEngineCandidates(boolean findByName) {
+        Map<ScriptEngine, Integer> matchPositionPerScriptEngineCandidate = new HashMap<>();
+        int matchPosition;
+        List<String> lookupCriteria;
+
         for (ScriptEngineFactory factory : new ScriptEngineManager().getEngineFactories()) {
-            for (String name : factory.getNames()) {
-                if (name.equalsIgnoreCase(scriptEngineLookup)) {
-                    return factory.getScriptEngine();
-                }
+            matchPosition = 0;
+            if (findByName) {
+                lookupCriteria = factory.getNames();
+            } else {
+                lookupCriteria = factory.getExtensions();
             }
-            for (String ext : factory.getExtensions()) {
-                String scriptEngineLookupLowercase = scriptEngineLookup.toLowerCase();
-                if (scriptEngineLookupLowercase.equalsIgnoreCase(ext.toLowerCase())) {
-                    return factory.getScriptEngine();
+
+            for (String criteria : lookupCriteria) {
+                if (criteria.equalsIgnoreCase(scriptEngineLookupName)) {
+                    matchPositionPerScriptEngineCandidate.put(factory.getScriptEngine(), matchPosition);
                 }
+                matchPosition++;
+            }
+        }
+
+        return matchPositionPerScriptEngineCandidate;
+    }
+
+    private ScriptEngine findBestScriptEngine(Map<ScriptEngine, Integer> scriptEngineCandidates) {
+        int minimumMatchingIndex = Integer.MAX_VALUE;
+        ScriptEngine bestScriptEngine = null;
+
+        for (Entry<ScriptEngine, Integer> candidate : scriptEngineCandidates.entrySet()) {
+            if (candidate.getValue() < minimumMatchingIndex) {
+                minimumMatchingIndex = candidate.getValue();
+                bestScriptEngine = candidate.getKey();
             }
         }
-        return null;
+
+        return bestScriptEngine;
     }
 
     /**
@@ -412,7 +448,7 @@ public static String readFile(File file) throws IOException {
     }
 
     public String getEngineName() {
-        return scriptEngineLookup;
+        return scriptEngineLookupName;
     }
 
     @Override
@@ -459,9 +495,9 @@ public String toString() {
 
     public String display() {
         String nl = System.lineSeparator();
-        return " { " + nl + "Script '" + getScriptName() + '\'' + nl + "\tscriptEngineLookup = '" + scriptEngineLookup +
-               '\'' + nl + "\tscript = " + nl + script + nl + "\tid = " + nl + id + nl + "\tparameters = " +
-               Arrays.toString(parameters) + nl + '}';
+        return " { " + nl + "Script '" + getScriptName() + '\'' + nl + "\tscriptEngineLookupName = '" +
+               scriptEngineLookupName + '\'' + nl + "\tscript = " + nl + script + nl + "\tid = " + nl + id + nl +
+               "\tparameters = " + Arrays.toString(parameters) + nl + '}';
     }
 
     public void overrideDefaultScriptName(String defaultScriptName) {

config/rm/settings.ini

@@ -126,6 +126,24 @@ pa.rm.ec2.properties=config/rm/deployment/ec2.properties
 # As a result, Nodes are not necessarily locked on the same host.
 pa.rm.nodes.lock.restoration=true
 
+# Defines if the node restoration feature is enabled.
+# When set to {@code true}:
+# - on RM startup the RM tries to look up the nodes that were present
+#   before the scheduler crashed
+# - the RM persists node information
+pa.rm.nodes.recovery=true
+
+# Insert a delay before a database node source update or any node operation
+# is executed. If set to 0, all database operation are executed synchronously.
+pa.rm.node.db.operations.delay=500
+
+# If set to {@code true}, and if {@link pa.rm.node.db.operations.delay} is not
+# set to 0, then node updates will be executed synchronously as much as possible.
+# In this case, node updates can still be postponed if the node creation is still
+# pending.
+#
+pa.rm.nodes.db.operations.update.synchronous=true
+
 # Defines if  the runtime (RT) have to be killed when the resource manager (RM) is shutdown.
 pa.rm.shutdown.kill.rt=true
 

config/scheduler/settings.ini

@@ -37,10 +37,12 @@ pa.scheduler.core.scheduledpoolnbthreads=2
 pa.scheduler.core.housekeeping.scheduledpoolnbthreads=5
 
 # Check for failed node frequency in second
+# Also used by the node to ping the scheduler after finishing a task
 pa.scheduler.core.nodepingfrequency=20
 
 # The scheduler will decide to restart a task, after a given tolerated number of failed attempts.
 # A value of zero means that the scheduler will restart a task after the first failure.
+# Also used by a node to retry to send the result of a task to the scheduler
 pa.scheduler.core.node.ping.attempts=1
 
 # Scheduler default policy full name
@@ -83,7 +85,7 @@ pa.scheduler.core.starttask.threadnumber=5
 pa.scheduler.core.listener.threadnumber=5
 
 # List of the scripts paths to execute at scheduler start. Paths are separated by a ';'.
-pa.scheduler.startscripts.paths=tools/load-examples.groovy
+pa.scheduler.startscripts.paths=tools/LoadPackages.groovy
 
 #-------------------------------------------------------
 #----------------   JOBS PROPERTIES   ------------------

config/security.java.policy-server

@@ -159,6 +159,8 @@ grant principal org.ow2.proactive.authentication.principals.GroupNamePrincipal "
     permission org.ow2.proactive.permissions.MethodCallPermission "org.ow2.proactive.scheduler.core.SchedulerFrontend.getMyAccountUsage";
     permission org.ow2.proactive.permissions.MethodCallPermission "org.ow2.proactive.scheduler.core.SchedulerFrontend.getGlobalSpaceURIs";
     permission org.ow2.proactive.permissions.MethodCallPermission "org.ow2.proactive.scheduler.core.SchedulerFrontend.getUserSpaceURIs";
+    permission org.ow2.proactive.permissions.MethodCallPermission "org.ow2.proactive.scheduler.core.SchedulerFrontend.getSchedulerProperties";
+
 };
 
 

gradle.properties

@@ -1,4 +1,4 @@
-#Tue, 24 Oct 2017 11:00:40 +0200
-programmingVersion=7.32.0-SNAPSHOT
-schedulingVersion=7.32.0-SNAPSHOT
-schedulingSerialver=732L
+#Wed, 15 Nov 2017 11:03:43 +0100
+programmingVersion=7.33.0-SNAPSHOT
+schedulingVersion=7.33.0-SNAPSHOT
+schedulingSerialver=733L

regression_tests.txt

@@ -22,4 +22,7 @@ TestWorkflowLoopJobs
 TestWorkflowLoopJobs2
 TestWorkflowReplicateJobs
 TestWorkflowReplicateJobs2
-TestWorkflowReplicateJobs3
+TestWorkflowReplicateJobs3
+RecoverLocalInfrastructureTest
+TaskReconnectionWithForkedTaskExecutorTest
+TaskReconnectionWithInProcessTaskExecutorTest

rest/rest-api/src/main/java/org/ow2/proactive_grid_cloud_portal/common/SchedulerRestInterface.java

@@ -1927,4 +1927,18 @@ public JobIdData copyAndResubmitWithGeneralInfo(@HeaderParam("sessionid") String
     public Map<Object, Object> getPortalConfiguration(@HeaderParam("sessionid") String sessionId)
             throws NotConnectedRestException, PermissionRestException;
 
+    /**
+     * returns scheduler properties
+     *
+     * @param sessionId
+     *            the session id associated to this new connection
+     * @return a map containing the properties
+     * @throws NotConnectedRestException
+     * @throws PermissionRestException
+     */
+    @GET
+    @Path("properties")
+    @Produces("application/json")
+    Map<String, Object> getSchedulerPropertiesFromSessionId(@HeaderParam("sessionid")
+    final String sessionId) throws NotConnectedRestException, PermissionRestException;
 }

rest/rest-cli/src/main/java/org/ow2/proactive_grid_cloud_portal/cli/CommandSet.java

@@ -78,6 +78,7 @@
 import org.ow2.proactive_grid_cloud_portal.cli.cmd.sched.GetJobStateCommand;
 import org.ow2.proactive_grid_cloud_portal.cli.cmd.sched.GetTaskOutputCommand;
 import org.ow2.proactive_grid_cloud_portal.cli.cmd.sched.GetTaskResultCommand;
+import org.ow2.proactive_grid_cloud_portal.cli.cmd.sched.InstallPackageCommand;
 import org.ow2.proactive_grid_cloud_portal.cli.cmd.sched.KillCommand;
 import org.ow2.proactive_grid_cloud_portal.cli.cmd.sched.KillJobCommand;
 import org.ow2.proactive_grid_cloud_portal.cli.cmd.sched.LinkRmCommand;
@@ -870,6 +871,17 @@ public class CommandSet {
                                                                          .commandClass(VersionCommand.class)
                                                                          .entry();
 
+    public static final CommandSet.Entry INSTALL_PACKAGE = CommandSetEntryBuilder.newInstance()
+                                                                                 .opt("pkg")
+                                                                                 .longOpt("installpackage")
+                                                                                 .description("Install the specified package in the catalog")
+                                                                                 .hasArgs(true)
+                                                                                 .numOfArgs(1)
+                                                                                 .argNames("package-dir")
+                                                                                 .jsCommand("installpackage(package-dir)")
+                                                                                 .commandClass(InstallPackageCommand.class)
+                                                                                 .entry();
+
     /**
      * CommandSet.Entry objects which are common to both Scheduler and Resource
      * Manager CLIs
@@ -903,7 +915,8 @@ public class CommandSet {
                                                                                      PUT_THIRD_PARTY_CREDENTIAL,
                                                                                      REMOVE_THIRD_PARTY_CREDENTIAL,
                                                                                      THIRD_PARTY_CREDENTIAL_KEY_SET,
-                                                                                     SCHEDULER_HELP, LIVE_LOG };
+                                                                                     SCHEDULER_HELP, LIVE_LOG,
+                                                                                     INSTALL_PACKAGE };
 
     /**
      * CommandSet.Entry objects which are specific to Resource Manager CLI