Skip to content

Commit

Permalink
Merge branch 'master' into rafael_htcondor_pilot
Browse files Browse the repository at this point in the history
  • Loading branch information
@rafaelfsilva
rafaelfsilva committed Jun 18, 2019
2 parents eabe556 + e61ad3a commit 4896cd6
Show file tree
Hide file tree
Showing 46 changed files with 2,599 additions and 700 deletions.
4 changes: 2 additions & 2 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,11 @@ before_install:
fi

script:
# building wrench
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
docker exec -w /home/wrench/wrench/build -it wrench cmake -DENABLE_BATSCHED=${BATSCHED} -DCMAKE_VERBOSE_MAKEFILE=ON -DCOVERAGE=1 ..;
#docker exec -w /home/wrench/wrench/build -it wrench make all unit_tests doc-gh;
docker exec -w /home/wrench/wrench/build -it wrench make all unit_tests;
docker exec -w /home/wrench/wrench/build -it wrench ./unit_tests;
travis_wait sleep infinity & docker exec -w /home/wrench/wrench/build -it wrench ./unit_tests;
fi

after_success:
Expand Down
3 changes: 3 additions & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ add_definitions("-Wall -Wno-unused-variable -Wno-unused-private-field")
if (ENABLE_BATSCHED)
add_definitions(-DENABLE_BATSCHED)
endif ()
if (ENABLE_MESSAGE_MANAGER)
add_definitions(-DMESSAGE_MANAGER)
endif ()

set(CMAKE_CXX_STANDARD 11)

Expand Down
295 changes: 147 additions & 148 deletions examples/simple-example/SimpleSimulatorBatch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,153 +15,152 @@

int main(int argc, char **argv) {

/*
* Declaration of the top-level WRENCH simulation object
*/
wrench::Simulation simulation;

/*
* Initialization of the simulation, which may entail extracting WRENCH-specific and
* Simgrid-specific command-line arguments that can modify general simulation behavior.
* Two special command-line arguments are --help-wrench and --help-simgrid, which print
* details about available command-line arguments.
*/
simulation.init(&argc, argv);

/*
* Parsing of the command-line arguments for this WRENCH simulation
*/
if (argc != 3) {
std::cerr << "Usage: " << argv[0] << " <xml platform file> <workflow file>" << std::endl;
exit(1);
}

/* The first argument is the platform description file, written in XML following the SimGrid-defined DTD */
char *platform_file = argv[1];
/* The second argument is the workflow description file, written in XML using the DAX DTD */
char *workflow_file = argv[2];


/* Reading and parsing the workflow description file to create a wrench::Workflow object */
std::cerr << "Loading workflow..." << std::endl;
wrench::Workflow workflow;
workflow.loadFromDAXorJSON(workflow_file, "1000Gf");
std::cerr << "The workflow has " << workflow.getNumberOfTasks() << " tasks " << std::endl;
std::cerr.flush();

/* Reading and parsing the platform description file to instantiate a simulated platform */
std::cerr << "Instantiating SimGrid platform..." << std::endl;
simulation.instantiatePlatform(platform_file);

/* Get a vector of all the hosts in the simulated platform */
std::vector<std::string> hostname_list = simulation.getHostnameList();

/* Instantiate a storage service, to be stated on some host in the simulated platform,
* and adding it to the simulation. A wrench::StorageService is an abstraction of a service on
* which files can be written and read. This particular storage service has a capacity
* of 10,000,000,000,000 bytes, and is a SimpleStorageService instance. The SimpleStorageService
* is a barebone storage service implementation provided by WRENCH.
* Throughout the simulation execution, input/output files of workflow tasks will be located
* in this storage service.
*/
std::string storage_host = hostname_list[(hostname_list.size() > 2) ? 2 : 1];
std::cerr << "Instantiating a SimpleStorageService on " << storage_host << "..." << std::endl;
auto storage_service = simulation.add(
new wrench::SimpleStorageService(storage_host, 10000000000000.0));

std::string wms_host = hostname_list[0];

/* Instantiate a batch service, to be started on some host in the simulation platform.
* A batch service is an abstraction of a compute service that corresponds to
* batch-scheduled platforms in which jobs are submitted to a queue and dispatched
* to compute nodes according to various scheduling algorithms.
* In this example, this particular batch service has no scratch storage space (size = 0).
* The last argument to the constructor
* shows how to configure particular simulated behaviors of the compute service via a property
* list. In this example, one specifies that the message that will be send to the service to
* terminate it will be 2048 bytes. See the documentation to find out all available
* configurable properties for each kind of service.
*/
std::shared_ptr<wrench::ComputeService> batch_service;

/* Add the batch service to the simulation, catching a possible exception */
try {
batch_service = simulation.add(new wrench::BatchComputeService(
wms_host, hostname_list, 0, {},
{{wrench::BatchComputeServiceMessagePayload::STOP_DAEMON_MESSAGE_PAYLOAD, 2048}}));

} catch (std::invalid_argument &e) {
std::cerr << "Error: " << e.what() << std::endl;
std::exit(1);
}

/* Create a list of compute services that will be used by the WMS */
std::set<std::shared_ptr<wrench::ComputeService>> compute_services;
compute_services.insert(batch_service);

/* Create a list of storage services that will be used by the WMS */
std::set<std::shared_ptr<wrench::StorageService>> storage_services;
storage_services.insert(storage_service);

/* Instantiate a WMS, to be stated on some host (wms_host), which is responsible
* for executing the workflow, and uses a scheduler (BatchStandardJobScheduler). That scheduler
* is instantiated with the batch service, the list of hosts available for running
* tasks, and also provided a pointer to the simulation object.
*
* The WMS implementation is in SimpleWMS.[cpp|h].
*/
std::cerr << "Instantiating a WMS on " << wms_host << "..." << std::endl;
auto wms = simulation.add(
new wrench::SimpleWMS(std::unique_ptr<wrench::BatchStandardJobScheduler>(
new wrench::BatchStandardJobScheduler(storage_service)),
nullptr, compute_services, storage_services, wms_host));

wms->addWorkflow(&workflow);

/* Instantiate a file registry service to be started on some host. This service is
* essentially a replica catalog that stores <file , storage service> pairs so that
* any service, in particular a WMS, can discover where workflow files are stored.
*/
std::string file_registry_service_host = hostname_list[(hostname_list.size() > 2) ? 1 : 0];
std::cerr << "Instantiating a FileRegistryService on " << file_registry_service_host << "..." << std::endl;
auto file_registry_service =
simulation.add(new wrench::FileRegistryService(file_registry_service_host));

/* It is necessary to store, or "stage", input files for the first task(s) of the workflow on some storage
* service, so that workflow execution can be initiated. The getInputFiles() method of the Workflow class
* returns the set of all workflow files that are not generated by workflow tasks, and thus are only input files.
* These files are then staged on the storage service.
*/
std::cerr << "Staging input files..." << std::endl;
std::map<std::string, wrench::WorkflowFile *> input_files = workflow.getInputFiles();
for (auto f : input_files) {
std::cerr << "---> " << f.second->getID() << "\n";
}
try {
simulation.stageFiles(input_files, storage_service);
} catch (std::runtime_error &e) {
std::cerr << "Exception: " << e.what() << std::endl;
return 0;
}

/* Launch the simulation. This call only returns when the simulation is complete. */
std::cerr << "Launching the Simulation..." << std::endl;
try {
simulation.launch();
} catch (std::runtime_error &e) {
std::cerr << "Exception: " << e.what() << std::endl;
/*
* Declaration of the top-level WRENCH simulation object
*/
wrench::Simulation simulation;

/*
* Initialization of the simulation, which may entail extracting WRENCH-specific and
* Simgrid-specific command-line arguments that can modify general simulation behavior.
* Two special command-line arguments are --help-wrench and --help-simgrid, which print
* details about available command-line arguments.
*/
simulation.init(&argc, argv);

/*
* Parsing of the command-line arguments for this WRENCH simulation
*/
if (argc != 3) {
std::cerr << "Usage: " << argv[0] << " <xml platform file> <workflow file>" << std::endl;
exit(1);
}

/* The first argument is the platform description file, written in XML following the SimGrid-defined DTD */
char *platform_file = argv[1];
/* The second argument is the workflow description file, written in XML using the DAX DTD */
char *workflow_file = argv[2];


/* Reading and parsing the workflow description file to create a wrench::Workflow object */
std::cerr << "Loading workflow..." << std::endl;
wrench::Workflow workflow;
workflow.loadFromDAXorJSON(workflow_file, "1000Gf");
std::cerr << "The workflow has " << workflow.getNumberOfTasks() << " tasks " << std::endl;
std::cerr.flush();

/* Reading and parsing the platform description file to instantiate a simulated platform */
std::cerr << "Instantiating SimGrid platform..." << std::endl;
simulation.instantiatePlatform(platform_file);

/* Get a vector of all the hosts in the simulated platform */
std::vector<std::string> hostname_list = simulation.getHostnameList();

/* Create a list of storage services that will be used by the WMS */
std::set<std::shared_ptr<wrench::StorageService>> storage_services;

/* Instantiate a storage service, to be stated on some host in the simulated platform,
* and adding it to the simulation. A wrench::StorageService is an abstraction of a service on
* which files can be written and read. This particular storage service has a capacity
* of 10,000,000,000,000 bytes, and is a SimpleStorageService instance. The SimpleStorageService
* is a barebone storage service implementation provided by WRENCH.
* Throughout the simulation execution, input/output files of workflow tasks will be located
* in this storage service.
*/
std::string storage_host = hostname_list[(hostname_list.size() > 2) ? 2 : 1];
std::cerr << "Instantiating a SimpleStorageService on " << storage_host << "..." << std::endl;
auto storage_service = simulation.add(
new wrench::SimpleStorageService(storage_host, 10000000000000.0));
storage_services.insert(storage_service);

std::string wms_host = hostname_list[0];

/* Create a list of compute services that will be used by the WMS */
std::set<std::shared_ptr<wrench::ComputeService>> compute_services;

/* Instantiate a batch service, to be started on some host in the simulation platform.
* A batch service is an abstraction of a compute service that corresponds to
* batch-scheduled platforms in which jobs are submitted to a queue and dispatched
* to compute nodes according to various scheduling algorithms.
* In this example, this particular batch service has no scratch storage space (size = 0).
* The last argument to the constructor
* shows how to configure particular simulated behaviors of the compute service via a property
* list. In this example, one specifies that the message that will be send to the service to
* terminate it will be 2048 bytes. See the documentation to find out all available
* configurable properties for each kind of service.
*/

/* Add the batch service to the simulation, catching a possible exception */
try {
auto batch_service = simulation.add(new wrench::BatchComputeService(
wms_host, hostname_list, 0, {},
{{wrench::BatchComputeServiceMessagePayload::STOP_DAEMON_MESSAGE_PAYLOAD, 2048}}));
compute_services.insert(batch_service);
} catch (std::invalid_argument &e) {
std::cerr << "Error: " << e.what() << std::endl;
std::exit(1);}



/* Instantiate a WMS, to be stated on some host (wms_host), which is responsible
* for executing the workflow, and uses a scheduler (BatchStandardJobScheduler). That scheduler
* is instantiated with the batch service, the list of hosts available for running
* tasks, and also provided a pointer to the simulation object.
*
* The WMS implementation is in SimpleWMS.[cpp|h].
*/
std::cerr << "Instantiating a WMS on " << wms_host << "..." << std::endl;
auto wms = simulation.add(
new wrench::SimpleWMS(std::unique_ptr<wrench::BatchStandardJobScheduler>(
new wrench::BatchStandardJobScheduler(storage_service)),
nullptr, compute_services, storage_services, wms_host));

wms->addWorkflow(&workflow);

/* Instantiate a file registry service to be started on some host. This service is
* essentially a replica catalog that stores <file , storage service> pairs so that
* any service, in particular a WMS, can discover where workflow files are stored.
*/
std::string file_registry_service_host = hostname_list[(hostname_list.size() > 2) ? 1 : 0];
std::cerr << "Instantiating a FileRegistryService on " << file_registry_service_host << "..." << std::endl;
auto file_registry_service =
simulation.add(new wrench::FileRegistryService(file_registry_service_host));

/* It is necessary to store, or "stage", input files for the first task(s) of the workflow on some storage
* service, so that workflow execution can be initiated. The getInputFiles() method of the Workflow class
* returns the set of all workflow files that are not generated by workflow tasks, and thus are only input files.
* These files are then staged on the storage service.
*/
std::cerr << "Staging input files..." << std::endl;
auto input_files = workflow.getInputFiles();
for (auto f : input_files) {
std::cerr << "---> " << f.second->getID() << "\n";
}
try {
simulation.stageFiles(input_files, storage_service);
} catch (std::runtime_error &e) {
std::cerr << "Exception: " << e.what() << std::endl;
return 0;
}

/* Launch the simulation. This call only returns when the simulation is complete. */
std::cerr << "Launching the Simulation..." << std::endl;
try {
simulation.launch();
} catch (std::runtime_error &e) {
std::cerr << "Exception: " << e.what() << std::endl;
return 0;
}
std::cerr << "Simulation done!" << std::endl;

/* Simulation results can be examined via simulation.output, which provides access to traces
* of events. In the code below, we retrieve the trace of all task completion events, print how
* many such events there are, and print some information for the first such event.
*/
std::vector<wrench::SimulationTimestamp<wrench::SimulationTimestampTaskCompletion> *> trace;
trace = simulation.getOutput().getTrace<wrench::SimulationTimestampTaskCompletion>();
std::cerr << "Number of entries in TaskCompletion trace: " << trace.size() << std::endl;
std::cerr << "Task in first trace entry: " << trace[0]->getContent()->getTask()->getID() << std::endl;

return 0;
}
std::cerr << "Simulation done!" << std::endl;

/* Simulation results can be examined via simulation.output, which provides access to traces
* of events. In the code below, we retrieve the trace of all task completion events, print how
* many such events there are, and print some information for the first such event.
*/
std::vector<wrench::SimulationTimestamp<wrench::SimulationTimestampTaskCompletion> *> trace;
trace = simulation.getOutput().getTrace<wrench::SimulationTimestampTaskCompletion>();
std::cerr << "Number of entries in TaskCompletion trace: " << trace.size() << std::endl;
std::cerr << "Task in first trace entry: " << trace[0]->getContent()->getTask()->getID() << std::endl;

return 0;
}

0 comments on commit 4896cd6

Please sign in to comment.