Skip to content

Commit

Permalink
(#157)
Browse files Browse the repository at this point in the history
Added a pilot job example
  • Loading branch information
henricasanova committed Apr 20, 2020
1 parent 71489c6 commit 1e8c86d
Show file tree
Hide file tree
Showing 18 changed files with 456 additions and 32 deletions.
1 change: 1 addition & 0 deletions conf/cmake/Examples.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ set(EXAMPLES_CMAKEFILES_TXT
examples/basic-examples/bare-metal-complex-job/CMakeLists.txt
examples/basic-examples/cloud-bag-of-tasks/CMakeLists.txt
examples/basic-examples/batch-bag-of-tasks/CMakeLists.txt
examples/basic-examples/batch-pilot-job/CMakeLists.txt
)

foreach (cmakefile ${EXAMPLES_CMAKEFILES_TXT})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
** the completion time of each workflow task is printed.
**
** Example invocation of the simulator for a 10-task workflow, with only WMS logging:
** ./bare-metal-bag-of-tasks-simulator 10 ./two_hosts.xml --wrench-no-logs --log=two_tasks_at_a_time_wms.threshold=info
** ./bare-metal-bag-of-tasks-simulator 10 ./two_hosts.xml --wrench-no-logs --log=custom_wms.threshold=info
**
** Example invocation of the simulator for a 6-task workflow with full logging:
** ./bare-metal-bag-of-tasks-simulator 6 ./two_hosts.xml
Expand Down Expand Up @@ -56,7 +56,7 @@ int main(int argc, char **argv) {

/* Parsing of the command-line arguments for this WRENCH simulation */
if (argc != 3) {
std::cerr << "Usage: " << argv[0] << " <an EVEN number of tasks> <xml platform file> [optional logging arguments]" << std::endl;
std::cerr << "Usage: " << argv[0] << " <an EVEN number of tasks> <xml platform file> [--wrench-no-logs --log=custom_wms.threshold=info]" << std::endl;
exit(1);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

#include "TwoTasksAtATimeWMS.h"

XBT_LOG_NEW_DEFAULT_CATEGORY(two_tasks_at_a_time_wms, "Log category for TwoTasksAtATimeWMS");
XBT_LOG_NEW_DEFAULT_CATEGORY(custom_wms, "Log category for TwoTasksAtATimeWMS");

namespace wrench {

Expand Down Expand Up @@ -87,6 +87,14 @@ namespace wrench {
auto expensive_ready_task = ready_tasks.at(ready_tasks.size() - 1);

/* Create a standard job for the tasks */
if (expensive_ready_task) {
WRENCH_INFO("Creating a job for tasks %s and %s",
cheap_ready_task->getID().c_str(),
expensive_ready_task->getID().c_str());
} else {
WRENCH_INFO("Creating a job for task %s",
cheap_ready_task->getID().c_str());
}

/* First, we need to create a map of file locations, stating for each file
* where is should be read/written */
Expand All @@ -106,12 +114,20 @@ namespace wrench {
service_specific_args[cheap_ready_task->getID()] = "4";
service_specific_args[expensive_ready_task->getID()] = "6";

WRENCH_INFO("Submitting the job, asking for %s cores for task %s, and "
"%s cores for task %s",
service_specific_args[cheap_ready_task->getID()].c_str(),
cheap_ready_task->getID().c_str(),
service_specific_args[expensive_ready_task->getID()].c_str(),
expensive_ready_task->getID().c_str());

/* Submit the job to the compute service */
job_manager->submitJob(standard_job, compute_service, service_specific_args);

/* Wait for a workflow execution event and process it. In this case we know that
* the event will be a StandardJobCompletionEvent, which is processed by the method
* processEventStandardJobCompletion() that this class overrides. */
WRENCH_INFO("Waiting for next event");
this->waitForAndProcessNextEvent();
}

Expand All @@ -127,9 +143,11 @@ namespace wrench {
void TwoTasksAtATimeWMS::processEventStandardJobCompletion(std::shared_ptr<StandardJobCompletedEvent> event) {
/* Retrieve the job that this event is for */
auto job = event->standard_job;
/* Retrieve the job's first (and in our case only) task */
auto task = job->getTasks().at(0);
WRENCH_INFO("Notified that a standard job has completed task %s", task->getID().c_str());
/* Retrieve the job's tasks */
for (auto const &task : job->getTasks()) {
WRENCH_INFO("Notified that a standard job has completed task %s",
task->getID().c_str());
}
}

/**
Expand All @@ -146,7 +164,7 @@ namespace wrench {
WRENCH_INFO("Notified that a standard job has failed for task %s with error %s",
task->getID().c_str(),
event->failure_cause->toString().c_str());
throw std::runtime_error("ABORTING DUE TO JOB FAILURE");
throw std::runtime_error("This should not happen in this example");
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
** the completion time of each workflow task is printed.
**
** Example invocation of the simulator for a 10-task workflow, with only WMS logging:
** ./bare-metal-chain-simulator 10 ./two_hosts.xml --wrench-no-logs --log=workflow_as_a_single_job_wms.threshold=info
** ./bare-metal-chain-simulator 10 ./two_hosts.xml --wrench-no-logs --log=custom_wms.threshold=info
**
** Example invocation of the simulator for a 5-task workflow with full logging:
** ./bare-metal-chain-simulator 5 ./two_hosts.xml
Expand Down Expand Up @@ -53,7 +53,7 @@ int main(int argc, char **argv) {

/* Parsing of the command-line arguments for this WRENCH simulation */
if (argc != 3) {
std::cerr << "Usage: " << argv[0] << " <number of tasks> <xml platform file> [optional logging arguments]" << std::endl;
std::cerr << "Usage: " << argv[0] << " <number of tasks> <xml platform file> [--wrench-no-logs --log=custom_wms.threshold=info]" << std::endl;
exit(1);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

#include "WorkflowAsAsingleJobWMS.h"

XBT_LOG_NEW_DEFAULT_CATEGORY(workflow_as_a_single_job_wms, "Log category for WorkflowAsAsingleJobWMS");
XBT_LOG_NEW_DEFAULT_CATEGORY(custom_wms, "Log category for WorkflowAsAsingleJobWMS");

namespace wrench {

Expand Down Expand Up @@ -88,14 +88,17 @@ namespace wrench {
}

/* Second, we create a job */
WRENCH_INFO("Creating a job for the entire workflow");
auto job = job_manager->createStandardJob(this->getWorkflow()->getTasks(), file_locations);

/* Submit the job to the compute service */
WRENCH_INFO("Submitting the job to the compute sercvice");
job_manager->submitJob(job, compute_service);

/* Wait for a workflow execution event and process it. In this case we know that
* the event will be a StandardJobCompletionEvent, which is processed by the method
* processEventStandardJobCompletion() that this class overrides. */
WRENCH_INFO("Waiting for next event");
this->waitForAndProcessNextEvent();

WRENCH_INFO("Workflow execution complete");
Expand All @@ -111,8 +114,9 @@ namespace wrench {
/* Retrieve the job that this event is for */
auto job = event->standard_job;
/* Retrieve the job's first (and in our case only) task */
auto task = job->getTasks().at(0);
WRENCH_INFO("Notified that a standard job has completed task %s", task->getID().c_str());
for (auto const &task : job->getTasks()) {
WRENCH_INFO("Notified that a standard job has completed task %s", task->getID().c_str());
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
*/


#ifndef WRENCH_ONE_TASK_AT_A_TIME_SCRATCH_H
#define WRENCH_ONE_TASK_AT_A_TIME_SCRATCH_H
#ifndef WRENCH_EXAMPLE_ONE_TASK_AT_A_TIME_SCRATCH_H
#define WRENCH_EXAMPLE_ONE_TASK_AT_A_TIME_SCRATCH_H

#include <wrench-dev.h>

Expand Down Expand Up @@ -42,4 +42,4 @@ namespace wrench {

};
}
#endif //WRENCH_ONE_TASK_AT_A_TIME_SCRATCH_H
#endif //WRENCH_EXAMPLE_ONE_TASK_AT_A_TIME_SCRATCH_H
4 changes: 2 additions & 2 deletions examples/basic-examples/bare-metal-chain/BareMetalChain.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
** the completion time of each workflow task is printed.
**
** Example invocation of the simulator for a 10-task workflow, with only WMS logging:
** ./bare-metal-chain-simulator 10 ./two_hosts.xml --wrench-no-logs --log=one_task_at_a_time_wms.threshold=info
** ./bare-metal-chain-simulator 10 ./two_hosts.xml --wrench-no-logs --log=custom_wms.threshold=info
**
** Example invocation of the simulator for a 5-task workflow with full logging:
** ./bare-metal-chain-simulator 5 ./two_hosts.xml
Expand Down Expand Up @@ -53,7 +53,7 @@ int main(int argc, char **argv) {

/* Parsing of the command-line arguments for this WRENCH simulation */
if (argc != 3) {
std::cerr << "Usage: " << argv[0] << " <number of tasks> <xml platform file> [optional logging arguments]" << std::endl;
std::cerr << "Usage: " << argv[0] << " <number of tasks> <xml platform file> [--wrench-no-logs --log=custom_wms.threshold=info]" << std::endl;
exit(1);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

#include "OneTaskAtATimeWMS.h"

XBT_LOG_NEW_DEFAULT_CATEGORY(one_task_at_a_time_wms, "Log category for OneTaskAtATimeWMS");
XBT_LOG_NEW_DEFAULT_CATEGORY(custom_wms, "Log category for OneTaskAtATimeWMS");

namespace wrench {

Expand Down Expand Up @@ -71,6 +71,7 @@ namespace wrench {
auto ready_task = this->getWorkflow()->getReadyTasks().at(0);

/* Create a standard job for the task */
WRENCH_INFO("Creating a job for task %s", ready_task->getID().c_str());

/* First, we need to create a map of file locations, stating for each file
* where is should be read/written */
Expand All @@ -82,6 +83,7 @@ namespace wrench {
auto standard_job = job_manager->createStandardJob(ready_task, file_locations);

/* Submit the job to the compute service */
WRENCH_INFO("Submitting the job to the compute service");
job_manager->submitJob(standard_job, compute_service);

/* Wait for a workflow execution event and process it. In this case we know that
Expand Down
4 changes: 2 additions & 2 deletions examples/basic-examples/bare-metal-complex-job/ComplexJob.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
** the completion time of each workflow task is printed.
**
** Example invocation of the simulator with only WMS logging:
** ./bare-metal-complex-job-simulator ./four_hosts.xml --wrench-no-logs --log=complex_job_wms.threshold=info
** ./bare-metal-complex-job-simulator ./four_hosts.xml --wrench-no-logs --log=custom_wms.threshold=info
**
** Example invocation of the simulator for a 5-task workflow with full logging:
** ./bare-metal-complex-job-simulator ./four_hosts.xml
Expand Down Expand Up @@ -53,7 +53,7 @@ int main(int argc, char **argv) {

/* Parsing of the command-line arguments for this WRENCH simulation */
if (argc != 2) {
std::cerr << "Usage: " << argv[0] << " <xml platform file> [optional logging arguments]" << std::endl;
std::cerr << "Usage: " << argv[0] << " <xml platform file> [--wrench-no-logs --log=custom_wms.threshold=info]" << std::endl;
exit(1);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

#include "ComplexJobWMS.h"

XBT_LOG_NEW_DEFAULT_CATEGORY(complex_job_wms, "Log category for ComplexJobWMS");
XBT_LOG_NEW_DEFAULT_CATEGORY(custom_wms, "Log category for ComplexJobWMS");

namespace wrench {

Expand Down Expand Up @@ -103,17 +103,20 @@ namespace wrench {
/* Let's create a set of file deletion operations to be performed
* AFTER the "post" file copies have been performed */
std::vector<std::tuple<WorkflowFile *, std::shared_ptr<FileLocation> >> cleanup_file_deletions;
cleanup_file_deletions.push_back(std::make_tuple(outfile_2, FileLocation::LOCATION(storage_service2)));
cleanup_file_deletions.emplace_back(outfile_2, FileLocation::LOCATION(storage_service2));

/* Create the standard job */
WRENCH_INFO("Creating a complex job with pre file copies, post file copies, and post file deletions to execute task %s", task->getID().c_str());
auto job = job_manager->createStandardJob({task}, file_locations, pre_file_copies, post_file_copies, cleanup_file_deletions);

/* Submit the job to the compute service */
WRENCH_INFO("Submitting job to the compute service");
job_manager->submitJob(job, compute_service);

/* Wait for a workflow execution event and process it. In this case we know that
* the event will be a StandardJobCompletionEvent, which is processed by the method
* processEventStandardJobCompletion() that this class overrides. */
WRENCH_INFO("Waiting for next event");
this->waitForAndProcessNextEvent();

WRENCH_INFO("Workflow execution complete");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
** the completion time of each workflow task is printed.
**
** Example invocation of the simulator for a 10-task workflow, with only WMS logging:
** ./batch-bag-of-tasks-simulator 10 ./four_hosts.xml --wrench-no-logs --log=two_tasks_at_a_time_batch_wms.threshold=info
** ./batch-bag-of-tasks-simulator 10 ./four_hosts.xml --wrench-no-logs --log=custom_wms.threshold=info
**
** Example invocation of the simulator for a 6-task workflow with full logging:
** ./batch-bag-of-tasks-simulator 6 ./four_hosts.xml
Expand Down Expand Up @@ -56,7 +56,7 @@ int main(int argc, char **argv) {

/* Parsing of the command-line arguments for this WRENCH simulation */
if (argc != 3) {
std::cerr << "Usage: " << argv[0] << " <a number of tasks> <xml platform file> [optional logging arguments]" << std::endl;
std::cerr << "Usage: " << argv[0] << " <a number of tasks> <xml platform file> [--wrench-no-logs --log=custom_wms.threshold=info]" << std::endl;
exit(1);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

#include "TwoTasksAtATimeBatchWMS.h"

XBT_LOG_NEW_DEFAULT_CATEGORY(two_tasks_at_a_time_batch_wms, "Log category for TwoTasksAtATimeBatchWMS");
XBT_LOG_NEW_DEFAULT_CATEGORY(custom_wms, "Log category for TwoTasksAtATimeBatchWMS");

namespace wrench {

Expand Down Expand Up @@ -78,7 +78,7 @@ namespace wrench {
}

/* Initialize and seed a RNG */
std::uniform_int_distribution<int> dist(0.0, 1000000000);
std::uniform_int_distribution<int> dist(0, 1000000000);
std::mt19937 rng(42);

/* While the workflow isn't done, repeat the main loop */
Expand Down
Loading

0 comments on commit 1e8c86d

Please sign in to comment.