Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Moved rack specific logic from the data layer to the DFS.
The DFSs require access to rack-related information. Previously, we had no way to access this data because the DFS is a field of the DataLayerManager which was storing the rack information. Affected modules: scheduling/flow sim/dfs Change-Id: I9fd063e3f3a5628c2dd4e65eb910ffce93e1eae1
- Loading branch information
Showing
10 changed files
with
183 additions
and
153 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| // The Firmament project | ||
| // Copyright (c) 2016 Ionel Gog <ionel.gog@cl.cam.ac.uk> | ||
|
|
||
| #include "sim/dfs/simulated_dfs.h" | ||
|
|
||
| namespace firmament { | ||
| namespace sim { | ||
|
|
||
| // Racks contain "between 29 and 31 computers" in Quincy test setup | ||
| DEFINE_uint64(machines_per_rack, 30, "Number of machines per rack"); | ||
|
|
||
| SimulatedDFS::SimulatedDFS() : unique_rack_id_(0) { | ||
| } | ||
|
|
||
| EquivClass_t SimulatedDFS::AddMachine(ResourceID_t machine_res_id) { | ||
| EquivClass_t rack_ec; | ||
| if (racks_with_spare_links_.size() > 0) { | ||
| // Assign the machine to a rack that has spare links. | ||
| rack_ec = *(racks_with_spare_links_.begin()); | ||
| } else { | ||
| // Add a new rack. | ||
| rack_ec = unique_rack_id_; | ||
| unique_rack_id_++; | ||
| CHECK(InsertIfNotPresent( | ||
| &rack_to_machine_res_, rack_ec, | ||
| unordered_set<ResourceID_t, boost::hash<ResourceID_t>>())); | ||
| racks_with_spare_links_.insert(rack_ec); | ||
| } | ||
| auto machines_in_rack = FindOrNull(rack_to_machine_res_, rack_ec); | ||
| CHECK_NOTNULL(machines_in_rack); | ||
| machines_in_rack->insert(machine_res_id); | ||
| // Erase the rack from the spare_links set if the rack is now full. | ||
| if (machines_in_rack->size() == FLAGS_machines_per_rack) { | ||
| racks_with_spare_links_.erase(rack_ec); | ||
| } | ||
| CHECK(InsertIfNotPresent(&machine_to_rack_ec_, machine_res_id, rack_ec)); | ||
| return rack_ec; | ||
| } | ||
|
|
||
| bool SimulatedDFS::RemoveMachine(ResourceID_t machine_res_id) { | ||
| bool rack_removed = false; | ||
| EquivClass_t rack_ec = GetRackForMachine(machine_res_id); | ||
| auto machines_in_rack = FindOrNull(rack_to_machine_res_, rack_ec); | ||
| CHECK_NOTNULL(machines_in_rack); | ||
| ResourceID_t res_id_tmp = machine_res_id; | ||
| machines_in_rack->erase(res_id_tmp); | ||
| if (machines_in_rack->size() == 0) { | ||
| // The rack doesn't have any machines left. Delete it! | ||
| // We have to delete empty racks because we're using the number | ||
| // of racks to efficiently find if there's a rack on which a task has no | ||
| // data. | ||
| rack_to_machine_res_.erase(rack_ec); | ||
| rack_removed = true; | ||
| } else { | ||
| racks_with_spare_links_.insert(rack_ec); | ||
| rack_removed = false; | ||
| } | ||
| machine_to_rack_ec_.erase(machine_res_id); | ||
| return rack_removed; | ||
| } | ||
|
|
||
| } // namespace sim | ||
| } // namespace firmament |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| // The Firmament project | ||
| // Copyright (c) 2016 Ionel Gog <ionel.gog@cl.cam.ac.uk> | ||
|
|
||
| #ifndef FIRMAMENT_SIM_DFS_SIMULATED_DFS_H | ||
| #define FIRMAMENT_SIM_DFS_SIMULATED_DFS_H | ||
|
|
||
| #include "base/common.h" | ||
| #include "base/types.h" | ||
| #include "misc/map-util.h" | ||
| #include "scheduling/data_layer_manager_interface.h" | ||
|
|
||
| namespace firmament { | ||
| namespace sim { | ||
|
|
||
| class SimulatedDFS { | ||
| public: | ||
| SimulatedDFS(); | ||
| virtual ~SimulatedDFS() {}; | ||
| /** | ||
| * Add num_blocks for a new task. | ||
| * @param td the descriptor of the new task | ||
| * @param num_blocks the number of blocks to add | ||
| * @param max_machine_spread the maximum number of machines over which | ||
| * the task's inputs should be spread. | ||
| */ | ||
| virtual void AddBlocksForTask(const TaskDescriptor& td, | ||
| uint64_t num_blocks, | ||
| uint64_t max_machine_spread) = 0; | ||
|
|
||
| /** | ||
| * Add a new machine to the DFS. | ||
| * @param machine_res_id the resource id of the new machine | ||
| * @return the id of the rack in which the machine is located | ||
| */ | ||
| virtual EquivClass_t AddMachine(ResourceID_t machine_res_id); | ||
| virtual void GetFileLocations(const string& file_path, | ||
| list<DataLocation>* locations) = 0; | ||
| /** | ||
| * Remove all the blocks of a task. | ||
| * @param task_id the id of the task for which to remove the blocks | ||
| */ | ||
| virtual void RemoveBlocksForTask(TaskID_t task_id) = 0; | ||
|
|
||
| /** | ||
| * Remove a machine from the DFS. This method also removes all the blocks from | ||
| * the machine and makes sure they're again replicated. | ||
| * @param machine_res_id the resource id of the machine to be removed | ||
| * @return true if the machine's rack no longer contains machines | ||
| */ | ||
| virtual bool RemoveMachine(ResourceID_t machine_res_id); | ||
|
|
||
| inline const unordered_set<ResourceID_t, boost::hash<ResourceID_t>>& | ||
| GetMachinesInRack(EquivClass_t rack_ec) { | ||
| auto machines_in_rack = FindOrNull(rack_to_machine_res_, rack_ec); | ||
| CHECK_NOTNULL(machines_in_rack); | ||
| return *machines_in_rack; | ||
| } | ||
| inline uint64_t GetNumRacks() { | ||
| return rack_to_machine_res_.size(); | ||
| } | ||
| inline void GetRackIDs(vector<EquivClass_t>* rack_ids) { | ||
| for (auto& rack_to_machines : rack_to_machine_res_) { | ||
| rack_ids->push_back(rack_to_machines.first); | ||
| } | ||
| } | ||
| inline EquivClass_t GetRackForMachine(ResourceID_t machine_res_id) { | ||
| EquivClass_t* rack_ec = | ||
| FindOrNull(machine_to_rack_ec_, machine_res_id); | ||
| CHECK_NOTNULL(rack_ec); | ||
| return *rack_ec; | ||
| } | ||
|
|
||
| private: | ||
| // Set storing the racks to which we can still connect machines. | ||
| unordered_set<EquivClass_t> racks_with_spare_links_; | ||
| // Map storing the machine resource ids associated with each rack. | ||
| unordered_map<EquivClass_t, | ||
| unordered_set<ResourceID_t, boost::hash<ResourceID_t>>> | ||
| rack_to_machine_res_; | ||
| // Map storing the rack EC associated with each machine. | ||
| unordered_map<ResourceID_t, EquivClass_t, boost::hash<ResourceID_t>> | ||
| machine_to_rack_ec_; | ||
| // Counter used to generate unique rack ids. | ||
| EquivClass_t unique_rack_id_; | ||
| }; | ||
|
|
||
| } // namespace sim | ||
| } // namespace firmament | ||
|
|
||
| #endif // FIRMAMENT_SIM_DFS_SIMULATED_DFS_H |
Oops, something went wrong.