diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index 37f2a0a5f4..b8ab6d596e 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -6,37 +6,32 @@ COMP Superscalar (COMPSs) is a programming model that aims to ease the developme
such as Clusters, Grids and Clouds. COMP Superscalar also features a runtime system that exploits the inherent parallelism of
applications at execution time.
-Release number: 2.9 (Jasmine)
-Release date: June-2021
+Release number: 2.10 (Kumquat)
+Release date: November-2021
-------------------------------
New features
* Runtime:
- - Support for nested tasks in agents deployment
- - New application time out functionality to enable a controlled finalization of applications before the wall_clock_limit
- - New flags to simplify application debugging (--keep_workingdir, --gen_coredump)
- - Support for loading the application environment from scripts
- - Support for tracing in agents environment.
- - Partial support for OSX systems.
+ - Support for http requests (@http decorator).
+ - Support for Java versions higher than 8.
+ - Enable grouping processes in MPI tasks with the processes_per_node flag.
+ - Partial support for OSX systems
* Python:
- - Support for Optional parameters and default values.
- - Support for monitoring task status from Jupyter notebooks.
- - Memory profile enabled
- - Support for Python workers cache.
- - Support for dynamic number of returns override at task invocation.
+ - PyArrow object serialization support
+ - Cache profiling enabled
- * DDS:
- - New methods and optimizations in DDS class.
Improvements:
- - Enabling the pass of extra flags for the queue system flags from enqueue_compss.
- - Support for multiple data layout in MPI tasks
- - Improvements in tracing system. More events and cfg
- - Enabling a flag to change extrae configuration file for python processes.
- - Configuration files for Barbora system.
+ - Fixes in managing directory parameters.
+ - Improvements in asynchronous file system operations.
+ - Improvements visualizing collections in task dependency graphs.
+ - Improvements and fixes in the support for different MPI versions (COMPSS_MPIRUN_TYPE)
+ - Improvements in tracing system. Fixes in events and cfgs
+ - Configuration files for Karolina, Mahti and CTE-AMD system.
- Several Bug fixes.
Known Limitations:
- - OSX support is limited to Java and Python2 without CPU affinity (require to execute with --cpu_affinity=disable)
+ - Collections are not supported in http tasks
+ - OSX support is limited to Java and Python without CPU affinity (require to execute with --cpu_affinity=disable). We have also detected issues when several python3 versions are installed in the system. Tracing is not available.
- Reduce operations can consume more disk space than the manually programmed n-ary reduction
- Objects used as task parameters must be serializable.
- Tasks that invoke Numpy and MKL may experience issues if a different MKL threads count is used in different tasks. This is due to the fact that MKL reuses threads in the different calls and it does not change the number of threads from one call to another. This can be also happen with other libraries implemented with OpenMP.
diff --git a/builders/buildscs b/builders/buildscs
index 1d5b80a4eb..6d9ca22efa 100755
--- a/builders/buildscs
+++ b/builders/buildscs
@@ -12,9 +12,11 @@
SCtarget=/apps/COMPSs/${BranchName^} # capitalize branch name
# Install on each enabled supercomputer
- ENABLED_SCS=("mn" "mt" "nord" "p9")
- SCS_SCRIPT_NAMES=("buildMN" "buildMT" "buildNord" "buildP9")
+ #ENABLED_SCS=("mn" "mt" "nord" "p9" "cte-amd")
+ #SCS_SCRIPT_NAMES=("buildMN" "buildMT" "buildNord" "buildP9" "buildCTE-AMD")
+ ENABLED_SCS=("mn" "mt" "p9" "cte-amd")
+ SCS_SCRIPT_NAMES=("buildMN" "buildMT" "buildP9" "buildCTE-AMD")
echo "Installing COMPSs branch ${BranchName} at SCs"
for i in "${!ENABLED_SCS[@]}"; do
diff --git a/builders/scs/p9/buildP9_aux b/builders/scs/p9/buildP9_aux
index b31fb904d0..07627a5d66 100755
--- a/builders/scs/p9/buildP9_aux
+++ b/builders/scs/p9/buildP9_aux
@@ -44,14 +44,13 @@
#######
# DEFINE MODULES
- modules="gcc openmpi boost java atlas/3.10.3 python/2.7.14"
+ modules="gcc/8.3.0 openmpi/4.0.1 boost java atlas/3.10.3 python/2.7.14"
unload_before_python="ignore"
py2_modules="python/2.7.14"
run_before_py3_modules="ignore"
py3_modules="python/3.6.5"
- run_before_py3_other_modules="ignore"
- py3_other_modules="ignore"
- #py3_other_modules="Python/3.7.0-foss-2018b"
+ run_before_py3_other_modules="module load cuda/10.2 cudnn/7.6.4 nccl/2.4.8 tensorrt/6.0.1 openmpi/4.0.1 scalapack/2.0.2 fftw/3.3.8 szip/2.1.1 ffmpeg/4.2.1 opencv/4.1.1"
+ py3_other_modules="python/3.7.4_ML"
# DEFINE CONFIGURATION PARAMETERS
#(Environment vars must be scaped to get the correct value if not it could take a value previous to load the modules)
diff --git a/changelog b/changelog
index c66310ba33..98fec29aa8 100644
--- a/changelog
+++ b/changelog
@@ -473,7 +473,43 @@ Known Limitations:
- Delete file calls for files used as input can produce a significant synchronization of the main code.
- Defining a parameter as OUT is only allowed for files and collections of objects with a default constructor.
-For further information, please refer to “COMPSs User Manual: Application development guide”.
+Release number: 2.10 (Kumquat)
+Release date: November-2021
+-------------------------------
+New features
+ * Runtime:
+ - Support for http requests (@http decorator).
+ - Support for Java versions higher than 8.
+ - Enable grouping processes in MPI tasks with the processes_per_node flag.
+ - Partial support for OSX systems
+
+ * Python:
+ - PyArrow object serialization support
+ - Cache profiling enabled
+
+Improvements:
+ - Fixes in managing directory parameters.
+ - Improvements in asynchronous file system operations.
+ - Improvements visualizing collections in task dependency graphs.
+ - Improvements and fixes in the support for different MPI versions (COMPSS_MPIRUN_TYPE)
+ - Improvements in tracing system. Fixes in events and cfgs
+ - Configuration files for Karolina, Mahti and CTE-AMD system.
+ - Several Bug fixes.
+
+Known Limitations:
+ - Collections are not supported in http tasks
+ - OSX support is limited to Java and Python without CPU affinity (require to execute with --cpu_affinity=disable). We have also detected issues when several python3 versions are installed in the system. Tracing is not available.
+ - Reduce operations can consume more disk space than the manually programmed n-ary reduction
+ - Objects used as task parameters must be serializable.
+ - Tasks that invoke Numpy and MKL may experience issues if a different MKL threads count is used in different tasks. This is due to the fact that MKL reuses threads in the different calls and it does not change the number of threads from one call to another. This can be also happen with other libraries implemented with OpenMP.
+ - C++ Objects declared as arguments in coarse-grain tasks must be passed as object pointers in order to have proper dependency management.
+ - Master as worker feature is not working for executions with persistent worker in C++.
+ - Coherence and concurrent writing in parameters annotated with the "Concurrent" direction must be managed by the underlying distributed storage system.
+ - Delete file calls for files used as input can produce a significant synchronization of the main code.
+ - Defining a parameter as OUT is only allowed for files and collections of objects with a default constructor.
+
+For further information, please refer to the COMPSs Documentation at:
+ https://compss-doc.readthedocs.io/en/stable/
Please find more details about the COMP Superscalar framework at:
- http://compss.bsc.es/
+ http://compss.bsc.es/
diff --git a/compss/programming_model/bindings/python/src/pycompss/interactive.py b/compss/programming_model/bindings/python/src/pycompss/interactive.py
index 338ebc03d4..dcdeca1955 100644
--- a/compss/programming_model/bindings/python/src/pycompss/interactive.py
+++ b/compss/programming_model/bindings/python/src/pycompss/interactive.py
@@ -83,7 +83,7 @@
STREAMING = False
LOG_PATH = tempfile.mkdtemp()
GRAPHING = False
-LINE_SEPARATOR = "******************************************************"
+LINE_SEPARATOR = "********************************************************"
# Initialize multiprocessing
@@ -393,7 +393,7 @@ def start(log_level="off", # type: str
# RUNTIME START
##############################################################
- print("* - Starting COMPSs runtime... *")
+ print("* - Starting COMPSs runtime... *")
sys.stdout.flush() # Force flush
compss_start(log_level, all_vars["trace"], True)
@@ -434,7 +434,7 @@ def start(log_level="off", # type: str
# MAIN EXECUTION
# let the user write an interactive application
- print("* - PyCOMPSs Runtime started... Have fun! *")
+ print("* - PyCOMPSs Runtime started... Have fun! *")
print(LINE_SEPARATOR)
# Emit the application start event (the 0 is in the stop function)
@@ -447,26 +447,26 @@ def __show_flower__():
:return: None
"""
- print(LINE_SEPARATOR) # NOSONAR # noqa
- print("*************** PyCOMPSs Interactive *****************") # NOSONAR # noqa
- print(LINE_SEPARATOR) # NOSONAR # noqa
- print("* .-~~-.--. _____ _______ *") # NOSONAR # noqa
- print("* : ) |____ \ / ___ \ *") # NOSONAR # noqa
- print("* .~ ~ -.\ /.- ~~ . ___) | | (___) | *") # NOSONAR # noqa
- print("* > `. .' < / ___/ \____ / *") # NOSONAR # noqa
- print("* ( .- -. ) | |___ _ / / *") # NOSONAR # noqa
- print("* `- -.-~ `- -' ~-.- -' |_____| |_| /__/ *") # NOSONAR # noqa
- print("* ( : ) _ _ .-: *") # NOSONAR # noqa
- print("* ~--. : .--~ .-~ .-~ } *") # NOSONAR # noqa
- print("* ~-.-^-.-~ \_ .~ .-~ .~ *") # NOSONAR # noqa
- print("* \ \ ' \ '_ _ -~ *") # NOSONAR # noqa
- print("* \`.\`. // *") # NOSONAR # noqa
- print("* . - ~ ~-.__\`.\`-.// *") # NOSONAR # noqa
- print("* .-~ . - ~ }~ ~ ~-.~-. *") # NOSONAR # noqa
- print("* .' .-~ .-~ :/~-.~-./: *") # NOSONAR # noqa
- print("* /_~_ _ . - ~ ~-.~-._ *") # NOSONAR # noqa
- print("* ~-.< *") # NOSONAR # noqa
- print(LINE_SEPARATOR) # NOSONAR # noqa
+ print(LINE_SEPARATOR) # NOSONAR # noqa
+ print("**************** PyCOMPSs Interactive ******************") # NOSONAR # noqa
+ print(LINE_SEPARATOR) # NOSONAR # noqa
+ print("* .-~~-.--. _____ __ ______ *") # NOSONAR # noqa
+ print("* : ) |____ \ / | / __ \ *") # NOSONAR # noqa
+ print("* .~ ~ -.\ /.- ~~ . ___) | /_ | | | | | *") # NOSONAR # noqa
+ print("* > `. .' < / ___/ | | | | | | *") # NOSONAR # noqa
+ print("* ( .- -. ) | |___ _ | | | |__| | *") # NOSONAR # noqa
+ print("* `- -.-~ `- -' ~-.- -' |_____| |_| |_| \______/ *") # NOSONAR # noqa
+ print("* ( : ) _ _ .-: *") # NOSONAR # noqa
+ print("* ~--. : .--~ .-~ .-~ } *") # NOSONAR # noqa
+ print("* ~-.-^-.-~ \_ .~ .-~ .~ *") # NOSONAR # noqa
+ print("* \ \ ' \ '_ _ -~ *") # NOSONAR # noqa
+ print("* \`.\`. // *") # NOSONAR # noqa
+ print("* . - ~ ~-.__\`.\`-.// *") # NOSONAR # noqa
+ print("* .-~ . - ~ }~ ~ ~-.~-. *") # NOSONAR # noqa
+ print("* .' .-~ .-~ :/~-.~-./: *") # NOSONAR # noqa
+ print("* /_~_ _ . - ~ ~-.~-._ *") # NOSONAR # noqa
+ print("* ~-.< *") # NOSONAR # noqa
+ print(LINE_SEPARATOR) # NOSONAR # noqa
def __print_setup__(verbose, all_vars):
diff --git a/compss/runtime/adaptors/engine/src/main/java/es/bsc/compss/types/data/accessid/RAccessId.java b/compss/runtime/adaptors/engine/src/main/java/es/bsc/compss/types/data/accessid/RAccessId.java
index 5ef61fc218..39731ad8d7 100644
--- a/compss/runtime/adaptors/engine/src/main/java/es/bsc/compss/types/data/accessid/RAccessId.java
+++ b/compss/runtime/adaptors/engine/src/main/java/es/bsc/compss/types/data/accessid/RAccessId.java
@@ -93,7 +93,7 @@ public int getRVersionId() {
* @return {@code true} if the source data must be preserved, {@code false} otherwise.
*/
public boolean isPreserveSourceData() {
- return this.readDataVersion.hasMoreReaders();
+ return this.readDataVersion.hasMoreReaders() || this.readDataVersion.getDataInstanceId().getVersionId() == 1;
}
@Override
diff --git a/compss/runtime/resources/commons/src/main/java/es/bsc/compss/connectors/AbstractConnector.java b/compss/runtime/resources/commons/src/main/java/es/bsc/compss/connectors/AbstractConnector.java
index e65cb5af9a..f712280ec2 100644
--- a/compss/runtime/resources/commons/src/main/java/es/bsc/compss/connectors/AbstractConnector.java
+++ b/compss/runtime/resources/commons/src/main/java/es/bsc/compss/connectors/AbstractConnector.java
@@ -16,7 +16,6 @@
*/
package es.bsc.compss.connectors;
-import es.bsc.compss.comm.Comm;
import es.bsc.compss.connectors.utils.CreationThread;
import es.bsc.compss.connectors.utils.DeadlineThread;
import es.bsc.compss.connectors.utils.DeletionThread;
@@ -28,11 +27,6 @@
import es.bsc.compss.types.resources.ShutdownListener;
import es.bsc.compss.types.resources.description.CloudMethodResourceDescription;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.ObjectInputStream;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -40,12 +34,8 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
-import jdk.nashorn.internal.parser.JSONParser;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import org.json.JSONArray;
-import org.json.JSONObject;
-import org.json.JSONTokener;
/**
diff --git a/compss/runtime/scripts/system/commons/version.sh b/compss/runtime/scripts/system/commons/version.sh
index c931f6b340..327cc076ba 100644
--- a/compss/runtime/scripts/system/commons/version.sh
+++ b/compss/runtime/scripts/system/commons/version.sh
@@ -2,7 +2,7 @@
# Displays version
###############################################
show_version() {
- echo "COMPSs version 2.9 Jasmine"
+ echo "COMPSs version 2.10 Kumquat"
echo " "
}
@@ -12,12 +12,12 @@ show_version() {
show_full_version() {
cat << EOF
- .-~~-.--. _____ _______
- : ) |____ \\ / ___ \\
- .~ ~ -.\\ /.- ~~ . ___) | | (___) |
- > \`. .' < / ___/ \\____ |
-( .- -. ) | |____ _ ____) |
- \`- -.-~ \`- -' ~-.- -' |______| |_| |______/
+ .-~~-.--. _____ _ ______
+ : ) |____ \\ / | / __ \\
+ .~ ~ -.\\ /.- ~~ . ___) | /_ | | | | |
+ > \`. .' < / ___/ | | | | | |
+( .- -. ) | |____ _ | | | |__| |
+ \`- -.-~ \`- -' ~-.- -' |______| |_| |_| \\______/
( : ) _ _ .-:
~--. : .--~ .-~ .-~ }
~-.-^-.-~ \\ .~ .-~ .~
@@ -37,9 +37,11 @@ EOF
###############################################
show_flower() {
cat << EOF
-Jasmine is a genus of shrubs and vines in the olive family (Oleaceae). The flowers are typically around 2.5 cm (0.98 in) in diameter. They are white or yellow in color, although in rare instances they can be slightly reddish. The flowers are borne in cymose clusters with a minimum of three flowers, though they can also be solitary on the ends of branchlets. Each flower has about four to nine petals, two locules, and one to four ovules. They have two stamens with very short filaments. The bracts are linear or ovated.
+Kumquats are a group of small fruit-bearing trees in the flowering plant family Rutaceae. They were previously classified as forming the now-historical genus Fortunella, or placed within Citrus, sensu lato.
+They are slow-growing evergreen shrubs or short trees that stand 2.5 to 4.5 meters (8 to 15 ft) tall, with dense branches, sometimes bearing small thorns. The leaves are dark glossy green, and the flowers are white, similar to other citrus flowers, and can be borne singly or clustered within the leaves' axils. Depending on size, the kumquat tree can produce hundreds or even thousands of fruits each year.
+
Check more information:
- https://en.wikipedia.org/wiki/Jasmine
+ https://en.wikipedia.org/wiki/Kumquat
EOF
}
@@ -50,20 +52,20 @@ EOF
show_recipe() {
cat << EOF
-Jasmine - Cocktail
+Kumquat Quest - Cocktail
-From: https://www.liquor.com/recipes/jasmine/
+From: https://cocktailsdistilled.com/recipe/kumquat-quest/
Ingredients:
- 1 1/2 oz Gin
- 1/4 oz Campari
- 1/4 oz Orange liqueur
- 3/4 oz Fresh lemon juice
- Garnish: Lemon twist
+ 2 oz Vodka
+ 1/2 oz Grand Marnier
+ 1/2 oz Lime Juice
+ Sugar, Demerara Brown
+ 3-4 Kumquats
Instructions:
- Place all ingredients in iced cocktail shaker strain. Shake and add sugar to taste. Strain into a chilled cocktail glass. Garnish with a lemon twist
+ Cut all the kumquats in half and put all the rest of the ingredients into a highball glass. Muddle all and add crushed ice. Float Grand Marnier.
EOF
}
diff --git a/pom.xml b/pom.xml
index 7a7dd60569..a35ba738e2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -130,7 +130,7 @@
compss-repo
BSC Workflows and Distributed Computing group
- http://compss.bsc.es/artifactory/libs-release
+ https://compss.bsc.es/artifactory/libs-release
diff --git a/tests/sources/local/agents/0.2_1_agent_stop/execution b/tests/sources/local/agents/0.2_1_agent_stop/execution
index 2faca78899..666e626e6b 100755
--- a/tests/sources/local/agents/0.2_1_agent_stop/execution
+++ b/tests/sources/local/agents/0.2_1_agent_stop/execution
@@ -431,6 +431,111 @@ echo "All Agents were properly shuts down using the flags --stop and --forward_t
# Copy LOG files
cp -rf "${COMPSs_log_folder}/${app_name}_0${retry_num}"/* "${target_log_folder}"
+
+
+# STARTING AGENT WITH --stop TO STOP IT WITH A FAILURE IN THE EXECUTION
+
+agent6_log_dir="${log_dir}/agent6/"
+agent6_output_log="${log_dir}/agent6.outputlog"
+agent6_error_log="${log_dir}/agent6.errorlog"
+
+compss_agent_start \
+ --hostname="COMPSsWorker06" \
+ --classpath="${base_app_dir}/${jar_name}" \
+ --log_dir=${agent6_log_dir} \
+ --rest_port="46601" \
+ --comm_port="46602" \
+ -d \
+ ${runcompss_opts} \
+ --project="${base_app_dir}/project.xml" \
+ --resources="${base_app_dir}/resources.xml" 1>"${agent6_output_log}" 2>"${agent6_error_log}" &
+
+agent_pid="$!"
+AGENT_PIDS="${agent_pid}"
+
+#CHECK WHETHER THE AGENTS ARE DEPLOYED
+retries="${NUM_RETRIES}"
+curl -XGET http://127.0.0.1:46601/COMPSs/test 1>/dev/null 2>/dev/null
+while [ ! "$?" == "0" ] && [ "${retries}" -gt "0" ]; do
+ sleep 1
+ retries=$((retries - 1 ))
+ curl -XGET http://127.0.0.1:46601/COMPSs/test 1>/dev/null 2>/dev/null
+done
+
+sleep 1
+
+RESULT=$(grep "test invoked" "${agent6_output_log}")
+if [ -z "${RESULT}" ]; then
+ echo "Agent failed to start" > >(tee -a "${error_log}")
+ exit 1
+fi
+echo "Agent started" > >(tee -a "${output_log}")
+
+# CHECK WHETHER THE PROJECT FILE HAS BEEN PROPERLY READ
+RESOURCES=$(curl -XGET http://127.0.0.1:46601/COMPSs/resources 2>/dev/null)
+ num_RESOURCES=$(echo "${RESOURCES}" | jq '.resources | length')
+if [ ! "${num_RESOURCES}" == "1" ]; then
+ echo "Wrong number of RESOURCE tags on the agent's output. Found .${num_RESOURCES}.; 1 expected." > >(tee -a "${error_log}")
+ exit 1
+fi
+RESOURCE=$(echo "${RESOURCES}" | jq '.resources[] | select(.name=="COMPSsWorker06")')
+if [ -z "${RESOURCE}" ]; then
+ echo "Wrong Resource on Agent; COMPSsWorker06 expected." > >(tee -a "${error_log}")
+ exit 1
+fi
+DESCRIPTION=$(echo "${RESOURCE}" | jq '.description ')
+PROCESSOR_COUNT=$(echo "${DESCRIPTION}" | jq '.processors | length')
+if [ ! "${PROCESSOR_COUNT}" == "1" ]; then
+ echo "Wrong number of Processors configured for the Agent. Found ${PROCESSOR_COUNT}; 1 expected." > >(tee -a "${error_log}")
+ exit 1
+fi
+CPU_COUNT=$(echo "${DESCRIPTION}" | jq '.processors[] | select(.name=="MainProcessor") | .units')
+if [ ! "${CPU_COUNT}" == "1" ]; then
+ echo "Wrong number of CPU cores configured for the Agent. Found ${CPU_COUNT}; 1 expected." > >(tee -a "${error_log}")
+ exit 1
+fi
+
+
+# INVOKING MAIN METHOD AND ENDING THE AGENT AFTERWARDS
+"${COMPSS_HOME}/Runtime/scripts/user/compss_agent_call_operation" \
+ --master_node="127.0.0.1" \
+ --master_port="46601" \
+ --stop \
+ es.bsc.compss.test.DemoClassError 1 > >(tee -a "${output_log}") 2> >(tee -a "${error_log}")
+
+if [ ! "$?" == "0" ]; then
+ echo "Could not invoke main method." > >(tee -a "${error_log}")
+ exit 1
+fi
+echo "Main function invoked" > >(tee -a "${output_log}")
+
+retries="${NUM_RETRIES}"
+while [ ! -f "${agent6_log_dir}/jobs/job1_NEW.out" ] && [ "${retries}" -gt "0" ]; do
+ sleep 1
+ retries=$((retries - 1 ))
+done
+if [ ! -f "${agent6_log_dir}/jobs/job1_NEW.out" ]; then
+ echo "Could not invoke main method." > >(tee -a "${error_log}")
+ exit 1
+fi
+method_name=$( grep "Executing" "${agent6_log_dir}/jobs/job1_NEW.out" | awk '{print $2}')
+if [ ! "${method_name}" == "main" ]; then
+ echo "Could not invoke main method." > >(tee -a "${error_log}")
+ exit 1
+fi
+echo "main method properly executed" > >(tee -a "${output_log}")
+
+sleep 1
+pid_info=""
+CONFIRMATION=""
+while [ -n "${pid_info}" ] || [ -z "${CONFIRMATION}" ]; do
+ sleep 1
+ pid_info=$(ps -ef | grep "${AGENT_PIDS}" | grep -v PID | grep -v grep)
+ CONFIRMATION=$(grep "Agent was shutdown" "${agent6_log_dir}/agent.log")
+done
+echo "The agent properly shuts when an error occurs on the execution" > >(tee -a "${output_log}")
+AGENT_PIDS=""
+
exit_value=0
#----------------------------------------------------------------------------------
diff --git a/tests/sources/local/agents/0.2_1_agent_stop/src/main/java/es/bsc/compss/test/DemoClassError.java b/tests/sources/local/agents/0.2_1_agent_stop/src/main/java/es/bsc/compss/test/DemoClassError.java
new file mode 100644
index 0000000000..8d2b4d589e
--- /dev/null
+++ b/tests/sources/local/agents/0.2_1_agent_stop/src/main/java/es/bsc/compss/test/DemoClassError.java
@@ -0,0 +1,30 @@
+package es.bsc.compss.test;
+
+public class DemoClassError {
+
+ public static void demoFunction(String repeatsStr) throws Exception {
+ throw new Exception("fake exception when executing to test agent stop on error");
+ }
+
+ public static void main(String[] args) {
+ System.out.println("Executing main function");
+ int repeats = Integer.parseInt(args[0]);
+ method(repeats);
+ }
+
+ private static void method(int repeats) {
+ for (int i = 0; i < repeats; i++) {
+ System.out.println("Iteration " + i);
+ addDelay();
+ }
+ }
+
+ public static void addDelay() {
+ System.out.println("Start time:" + System.currentTimeMillis());
+ try {
+ Thread.sleep(1000);
+ } catch (Exception e) {
+ }
+ System.out.println("End time:" + System.currentTimeMillis());
+ }
+}
diff --git a/tests/sources/local/python/0_files_duplicated/deploy b/tests/sources/local/python/0_files_duplicated/deploy
new file mode 100755
index 0000000000..91a18f61d9
--- /dev/null
+++ b/tests/sources/local/python/0_files_duplicated/deploy
@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+ # Obtain parameter
+ source_base_dir=$1
+ target_base_dir=$2
+
+ # Copy files
+ cp "${source_base_dir}"/execution "${target_base_dir}"/execution
+ cp "${source_base_dir}"/result "${target_base_dir}"/result
+
+ cp -r "${source_base_dir}"/src "${target_base_dir}"
+
+ cp "${source_base_dir}"/project.xml "${target_base_dir}"/project.xml
+ cp "${source_base_dir}"/resources.xml "${target_base_dir}"/resources.xml
\ No newline at end of file
diff --git a/tests/sources/local/python/0_files_duplicated/execution b/tests/sources/local/python/0_files_duplicated/execution
new file mode 100755
index 0000000000..f4f76c5ff6
--- /dev/null
+++ b/tests/sources/local/python/0_files_duplicated/execution
@@ -0,0 +1,87 @@
+#!/bin/bash
+
+ # Obtain parameters
+ COMPSs_exec=$1
+ comm=$2
+ runcompss_opts=$3
+ base_app_dir=$4
+ COMPSs_log_folder=$5
+ target_log_folder=$6
+ retry_num=$7
+ execution_envs=$8
+
+ # Global variables
+ exit_value=1
+ app_name="files_duplicated.py"
+
+ #----------------------------------------------------------------------------------
+ # Generate Project.xml according to system
+ sed -i '//c'"${COMPSS_HOME}"'<\/InstallDir>' "${base_app_dir}"/project.xml
+
+ #----------------------------------------------------------------------------------
+ # Create all working dir folders
+ mkdir -p /tmp/COMPSsWorker01/
+
+ #----------------------------------------------------------------------------------
+ # Run application
+ echo
+ echo "*** RUNNING PYTHON APPLICATION FILES"
+
+ for python_interpreter in ${execution_envs}; do
+ echo " - Running with Interpreter: ${python_interpreter}"
+ output_log="${target_log_folder}/${app_name}_${python_interpreter}_0${retry_num}.outputlog"
+ error_log="${target_log_folder}/${app_name}_${python_interpreter}_0${retry_num}.errorlog"
+ specific_log_dir="${COMPSs_log_folder}/${app_name}_${python_interpreter}_0${retry_num}"
+ exec_target_log_folder="${target_log_folder}/${app_name}_${python_interpreter}_0${retry_num}"
+ mkdir -p "${exec_target_log_folder}"
+ expectedTime=80s
+
+ python_mpi_worker=""
+ if [ "$python_interpreter" == 'python2_mpi' ] || [ "$python_interpreter" == 'python3_mpi' ]; then
+ echo "WARN: Test not requiered for ${python_interpreter}" | tee -a "${output_log}"
+ continue
+ fi
+
+ # shellcheck disable=SC2086
+ timeout --signal=2 $expectedTime "${COMPSs_exec}" \
+ --project="${base_app_dir}"/project.xml \
+ --resources="${base_app_dir}"/resources.xml \
+ \
+ --comm="$comm" \
+ \
+ --debug \
+ ${runcompss_opts} \
+ \
+ --lang=python \
+ --python_interpreter="${python_interpreter}" \
+ ${python_mpi_worker} \
+ --pythonpath="${base_app_dir}"/src \
+ --specific_log_dir="${specific_log_dir}" \
+ \
+ "${base_app_dir}/src/${app_name}" > >(tee "${output_log}") 2> >(tee "${error_log}" >&2)
+
+ # Check timeout
+ if [[ "$?" = "124" ]]; then
+ # Copy LOG files
+ cp -rf "${specific_log_dir}"/* "${exec_target_log_folder}"
+
+ # Report time out
+ echo "[WARNING] Python Application timed out" >> "${output_log}"
+ exit_value=124
+ else
+ # Copy LOG files
+ cp -rf "${specific_log_dir}"/* "${exec_target_log_folder}"
+
+ # Check result
+ "${base_app_dir}"/result "${output_log}" "${error_log}" "${exec_target_log_folder}" "${base_app_dir}"
+ exit_value=$?
+ fi
+
+ if [ ${exit_value} -ne 0 ]; then
+ exit ${exit_value}
+ fi
+ done
+
+ #----------------------------------------------------------------------------------
+ # Return execution status
+ exit ${exit_value}
diff --git a/tests/sources/local/python/0_files_duplicated/project.xml b/tests/sources/local/python/0_files_duplicated/project.xml
new file mode 100644
index 0000000000..4e34c5b6ec
--- /dev/null
+++ b/tests/sources/local/python/0_files_duplicated/project.xml
@@ -0,0 +1,8 @@
+
+
+
+
+ /opt/COMPSs/
+ /tmp/COMPSsWorker01/
+
+
diff --git a/tests/sources/local/python/0_files_duplicated/resources.xml b/tests/sources/local/python/0_files_duplicated/resources.xml
new file mode 100644
index 0000000000..8bd4d851aa
--- /dev/null
+++ b/tests/sources/local/python/0_files_duplicated/resources.xml
@@ -0,0 +1,28 @@
+
+
+
+
+ 4
+
+
+
+
+
+
+
+ 43001
+ 43002
+
+
+
+
+
+ sequential
+
+
+
+ sshtrilead
+
+
+
+
diff --git a/tests/sources/local/python/0_files_duplicated/result b/tests/sources/local/python/0_files_duplicated/result
new file mode 100755
index 0000000000..859db6aa65
--- /dev/null
+++ b/tests/sources/local/python/0_files_duplicated/result
@@ -0,0 +1,102 @@
+#!/bin/bash -e
+
+ # Obtain parameters
+ output_file=$1
+ error_file=$2
+ target_log_folder=$3
+ base_app_dir=$4
+
+ # Log files
+ runtime_log="${target_log_folder}/runtime.log"
+ resources_log="${target_log_folder}/resources.log"
+
+ #----------------------------------------------------------------------------------
+ # Check output standard out status
+ if [ -f "${output_file}" ]; then
+ result_expected="------------------------------------------------------------"
+ test_result=$(tail -1 "${output_file}")
+ if [ "${result_expected}" != "${test_result}" ]; then
+ echo "SOME ERROR WAS FOUND. CHECK ERRORS ABOVE." | tee -a "${output_file}"
+ exit 1
+ fi
+ else
+ echo "[ERROR] Output file not found" | tee -a "${output_file}"
+ exit 1
+ fi
+ # Check that ErrorManager has not raised any ERROR or FATAL message
+ if [ -f "${error_file}" ]; then
+ if grep -E "\\[ERRMGR\\].*(ERROR|FATAL|Shutting down COMPSs)" "${error_file}"; then
+ echo "ERROR: ErrorManager has raised a fatal error" | tee -a "${output_file}"
+ exit 1
+ fi
+ else
+ # Empty error file, no exceptions could have been raised
+ :
+ fi
+ # Check that the Runtime has not registered any fatal error
+ if [ -f "${runtime_log}" ]; then
+ if grep -E "ErrorManager\\].*\\[ERRMGR\\].*Error detected. Shutting down COMPSs" "${runtime_log}"; then
+ echo "ERROR: ErrorManager has registered a fatal error in the runtime.log" | tee -a "${output_file}"
+ exit 1
+ fi
+ else
+ echo "[ERROR] runtime.log not found" | tee -a "${output_file}"
+ exit 1
+ fi
+
+ #----------------------------------------------------------------------------------
+ # Grep for unit test errors (ERROR or FAIL)
+ errors=$(cat "${output_file}" "${error_file}" | grep "ERROR" | cat)
+ fails=$(cat "${output_file}" "${error_file}" | grep "FAIL" | cat)
+ if [ "${errors}" != "" ] || [ "${fails}" != "" ]; then
+ echo "ERROR: UNIT TEST RAISED AN ERROR." | tee -a "${output_file}"
+ exit 1
+ fi
+
+ #----------------------------------------------------------------------------------
+ # Check files consolidation
+ if [ -f "${base_app_dir}"/infile ]; then
+ content=$(cat "${base_app_dir}"/infile)
+ if [ "${content}" != "IN FILE CONTENT" ]; then
+ echo "ERROR: infile FILE CONTENTS ARE NOT AS EXPECTED AFTER EXECUTION." | tee -a "${output_file}"
+ exit 2
+ fi
+ fi
+
+ if [ -f "${base_app_dir}"/inoutfile ]; then
+ content=$(cat "${base_app_dir}"/inoutfile)
+ content="$(echo "$content" | tr -d '\n')"
+ if [ "${content}" != "INOUT FILE CONTENT===> INOUT FILE ADDED CONTENT" ]; then
+ echo "ERROR: inoutfile FILE CONTENTS ARE NOT AS EXPECTED AFTER EXECUTION." | tee -a "${output_file}"
+ exit 3
+ fi
+ fi
+
+ if [ -f "${base_app_dir}"/outfile ]; then
+ content=$(cat "${base_app_dir}"/outfile)
+ if [ "${content}" != "OUT FILE CONTENT" ]; then
+ echo "ERROR: outfile FILE CONTENTS ARE NOT AS EXPECTED AFTER EXECUTION." | tee -a "${output_file}"
+ exit 4
+ fi
+ fi
+
+ if [ -f "${base_app_dir}"/inwork ]; then
+ content=$(cat "${base_app_dir}"/inwork)
+ if [ "${content}" != "Before the task " ]; then
+ echo "ERROR: inwork FILE CONTENTS ARE NOT AS EXPECTED AFTER EXECUTION." | tee -a "${output_file}"
+ exit 5
+ fi
+ fi
+
+ if [ -f "${base_app_dir}"/outwork ]; then
+ content=$(cat "${base_app_dir}"/outwork)
+ content="$(echo "$content" | tr -d '\n')"
+ if [ "${content}" != "Before the task ===> INOUT FILE ADDED CONTENT" ]; then
+ echo "ERROR: outwork FILE CONTENTS ARE NOT AS EXPECTED AFTER EXECUTION." | tee -a "${output_file}"
+ exit 6
+ fi
+ fi
+
+ #----------------------------------------------------------------------------------
+ # All ok
+ exit 0
\ No newline at end of file
diff --git a/tests/sources/local/python/0_files_duplicated/src/files_duplicated.py b/tests/sources/local/python/0_files_duplicated/src/files_duplicated.py
new file mode 100644
index 0000000000..48b1a592a3
--- /dev/null
+++ b/tests/sources/local/python/0_files_duplicated/src/files_duplicated.py
@@ -0,0 +1,22 @@
+#!/usr/bin/python
+
+# -*- coding: utf-8 -*-
+
+"""
+PyCOMPSs Testbench
+========================
+"""
+
+# Imports
+import unittest
+
+from modules.testFiles import testFiles
+
+
+def main():
+ suite = unittest.TestLoader().loadTestsFromTestCase(testFiles)
+ unittest.TextTestRunner(verbosity=2).run(suite)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/sources/local/python/0_files_duplicated/src/infile b/tests/sources/local/python/0_files_duplicated/src/infile
new file mode 100644
index 0000000000..43d92ec340
--- /dev/null
+++ b/tests/sources/local/python/0_files_duplicated/src/infile
@@ -0,0 +1 @@
+IN FILE CONTENT
\ No newline at end of file
diff --git a/tests/sources/local/python/0_files_duplicated/src/inoutfile b/tests/sources/local/python/0_files_duplicated/src/inoutfile
new file mode 100644
index 0000000000..af33f54592
--- /dev/null
+++ b/tests/sources/local/python/0_files_duplicated/src/inoutfile
@@ -0,0 +1 @@
+INOUT FILE CONTENT
\ No newline at end of file
diff --git a/tests/sources/local/python/0_files_duplicated/src/modules/__init__.py b/tests/sources/local/python/0_files_duplicated/src/modules/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/sources/local/python/0_files_duplicated/src/modules/testFiles.py b/tests/sources/local/python/0_files_duplicated/src/modules/testFiles.py
new file mode 100644
index 0000000000..00372539a1
--- /dev/null
+++ b/tests/sources/local/python/0_files_duplicated/src/modules/testFiles.py
@@ -0,0 +1,158 @@
+#!/usr/bin/python
+
+# -*- coding: utf-8 -*-
+
+"""
+PyCOMPSs Testbench Tasks
+========================
+"""
+
+# Imports
+import unittest
+import os
+from pycompss.api.task import task
+from pycompss.api.parameter import *
+
+
+class testFiles(unittest.TestCase):
+
+ @task(fin=FILE, fin2=FILE, returns=2)
+ def fileInDpulicated(self, fin, fin2, name):
+ print("TEST FILE NAME")
+ if fin.endswith(name):
+ print("The file name is OK: " + str(fin))
+ else:
+ raise Exception("FILENAME NOT AS EXPECTED: " + str(fin) + " != " + str(name))
+ fin_d = open(fin, 'r')
+ content = fin_d.read()
+ print("- Inout file content:\n", content)
+ if fin2.endswith(name):
+ print("The file name is OK: " + str(fin2))
+ else:
+ raise Exception("FILENAME NOT AS EXPECTED: " + str(fin2) + " != " + str(name))
+ fin2_d = open(fin2, 'r')
+ content2 = fin2_d.read()
+ print("- Inout file content:\n", content2)
+ return content, content2
+
+ @task(fin=FILE, fin2=FILE_INOUT, returns=2)
+ def fileInInoutDpulicated(self, fin, fin2, name):
+ print("TEST FILE NAME")
+ if fin.endswith(name):
+ print("The file name is OK: " + str(fin))
+ else:
+ raise Exception("FILENAME NOT AS EXPECTED: " + str(fin) + " != " + str(name))
+ fin_d = open(fin, 'r')
+ content = fin_d.read()
+ print("- In file content:\n", content)
+ if fin2.endswith(name):
+ print("The file name is OK: " + str(fin2))
+ else:
+ raise Exception("FILENAME NOT AS EXPECTED: " + str(fin2) + " != " + str(name))
+ fin2_d = open(fin2, 'r+')
+ content2 = fin2_d.read()
+ print("- Inout file content:\n", content2)
+ # Add some content
+ content2 += "\n===> INOUT FILE ADDED CONTENT"
+ fin2_d.write("\n===> INOUT FILE ADDED CONTENT")
+ print("- Inout file content after modification:\n", content2)
+ # Close and return with the modification
+ fin2_d.close()
+ return content, content2
+
+ @task(fin=FILE, fout=FILE_OUT, returns=2)
+ def fileInOutDuplicated(self, fin, fout, name, content_out):
+ if fin.endswith(name):
+ print("The file name is OK: " + str(fin))
+ else:
+ raise Exception("FILENAME NOT AS EXPECTED: " + str(fin) + " != " + str(name))
+ fin_d = open(fin, 'r')
+ content = fin_d.read()
+ print("- In file content:\n", content)
+ if fout.endswith(name):
+ print("The file name is OK: " + str(fout))
+ else:
+ raise Exception("FILENAME NOT AS EXPECTED: " + str(fout) + " != " + str(name))
+
+ # Open the file for writting and write some content
+ with open(fout, 'w') as fout_d:
+ fout_d.write(content_out)
+ print("- Out file content added:\n", content_out)
+ return content, content_out
+
+
+ @task(target_direction=IN, fout=FILE_OUT, returns=str)
+ def fileOut(self, fout, content_out):
+ print("TEST FILE OUT")
+ # Open the file for writting and write some content
+ with open(fout, 'w') as fout_d:
+ fout_d.write(content_out)
+ print("- Out file content added:\n", content_out)
+ return content_out
+
+ def testFileINDuplic(self):
+ """ Test FILE_IN """
+ from pycompss.api.api import compss_wait_on
+ if not os.path.exists("id1"):
+ os.mkdir("id1")
+ if not os.path.exists("id2"):
+ os.mkdir("id2")
+ fin1 = "id1/infile.txt"
+ fin2 = "id2/infile.txt"
+
+ content1 = "IN FILE CONTENT 1"
+ content2 = "IN FILE CONTENT 2"
+ self.fileOut(fin1, content1)
+ self.fileOut(fin2, content2)
+ res1,res2 = self.fileInDpulicated(fin1, fin2, "infile.txt")
+ res1 = compss_wait_on(res1)
+ res2 = compss_wait_on(res2)
+ self.assertEqual(res1, content1, "strings are not equal: {}, {}".format(res1, content1))
+ self.assertEqual(res1, content1, "strings are not equal: {}, {}".format(res1, content1))
+
+ def testFileIN_OUTDuplic(self):
+ """ Test FILE_OUT """
+ from pycompss.api.api import compss_wait_on
+ from pycompss.api.api import compss_open
+ if not os.path.exists("id1"):
+ os.mkdir("id1")
+ if not os.path.exists("id2"):
+ os.mkdir("id2")
+ fin = "id1/outfile"
+ fout = "id2/outfile"
+ content_1 = "IN FILE CONTENT"
+ content_2 = "OUT FILE CONTENT"
+ self.fileOut(fin, content_1)
+ res1, res2 = self.fileInOutDuplicated(fin, fout, "outfile", content_2)
+ res1 = compss_wait_on(res1)
+ with compss_open(fout, 'r') as fout_r:
+ content_r = fout_r.read()
+ # The final file is only stored after the execution.
+ # During the execution, you have to use the compss_open, which will
+ # provide the real file where the output file is.
+ # fileInFolder = os.path.exists(fout)
+ # self.assertTrue(fileInFolder, "FILE_OUT is not in the final location")
+ self.assertEqual(res1, content_1, "strings are not equal: {}, {}".format(res1, content_1))
+ self.assertEqual(content_r, content_2, "strings are not equal: {}, {}".format(content_r, content_2))
+
+ def testFileIN_INOUTDuplic(self):
+ """ Test FILE_INOUT """
+ from pycompss.api.api import compss_wait_on
+ from pycompss.api.api import compss_open
+ if not os.path.exists("id1"):
+ os.mkdir("id1")
+ if not os.path.exists("id2"):
+ os.mkdir("id2")
+ fin = "id1/inoutfile"
+ finout = "id2/inoutfile"
+ content1 = "INOUT FILE CONTENT 1"
+ content2 = "INOUT FILE CONTENT 2"
+ self.fileOut(fin, content1)
+ self.fileOut(finout, content2)
+ res1, res2 = self.fileInInoutDpulicated(fin, finout, "inoutfile")
+ res1 = compss_wait_on(res1)
+ with compss_open(finout, 'r') as finout_r:
+ content_r = finout_r.read()
+ content2 += "\n===> INOUT FILE ADDED CONTENT"
+ self.assertEqual(res1, content1, "strings 1 are not equal: {}, {}".format(res1, content1))
+ self.assertEqual(content_r, content2, "strings 2 are not equal: {}, {}".format(content_r, content2))
diff --git a/utils/scripts/update_version/deploy_tag_to_AMD b/utils/scripts/update_version/deploy_tag_to_AMD
new file mode 100755
index 0000000000..6b05217eed
--- /dev/null
+++ b/utils/scripts/update_version/deploy_tag_to_AMD
@@ -0,0 +1,36 @@
+#!/bin/bash -e
+
+ usage() {
+ exitValue=$1
+
+ echo " "
+ echo "Usage $0 "
+ echo " "
+
+ exit "$exitValue"
+ }
+
+
+ #=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+ # MAIN
+ #=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+
+ # Check parameters
+ if [ $# -ne 1 ]; then
+ echo "ERROR: Incorrect number of parameters"
+ usage 1
+ fi
+
+ # Set parameters
+ version=$1
+ buildFolder="cte-amd"
+ buildScript="buildCTE-AMD"
+ MNuser=compss
+ MNendpoint=amdlogin1.bsc.es
+
+ # Call generic script
+ ./deploy_tag_to_SC "${version}" ${buildFolder} ${buildScript} ${MNuser} ${MNendpoint}
+
+ # END
+ exit 0
+
diff --git a/utils/scripts/update_version/deploy_tag_to_SC b/utils/scripts/update_version/deploy_tag_to_SC
index 06a196ba4a..4cdb6fe440 100755
--- a/utils/scripts/update_version/deploy_tag_to_SC
+++ b/utils/scripts/update_version/deploy_tag_to_SC
@@ -28,7 +28,7 @@
install() {
echo "- Install to SC"
cd "${trunk_dir}"
- mvn -U clean install
+ #mvn -U clean install
cd "${buildersDir}"
./"${buildScript}" "${SCuser}" "/home/bsc19/${SCuser}/tmpCOMPSs/" "/apps/COMPSs/$version"
diff --git a/utils/scripts/update_version/main_release b/utils/scripts/update_version/main_release
index 7d14e0951f..899effd4d0 100755
--- a/utils/scripts/update_version/main_release
+++ b/utils/scripts/update_version/main_release
@@ -48,7 +48,7 @@ usage() {
./deploy_tag_to_Nord "${version}"
./deploy_tag_to_MT "${version}"
./deploy_tag_to_P9 "${version}"
-
+ ./deploy_tag_to_AMD "${version}"
# END
echo "ALL SUCCESS!"
exit 0