Permalink
Browse files

Moved common types to RuntimeTypes, moved DeviceManager into runtime …

…namespace.
  • Loading branch information...
gcatron committed Feb 11, 2019
1 parent 517f4de commit 1a21851de769ea7e2c7f50803e8018e99d99b471
@@ -76,7 +76,7 @@ void dispatchClassify(unsigned int id, HostManager *hostManager,
"resnet50" + std::to_string(id), std::move(ctx),
[id, path, &returned, &finished](RunIdentifierTy, ResultCode r,
std::unique_ptr<Context> ctx) {
if (r == ResultCode::Cancelled) {
if (r == ResultCode::Canceled) {
llvm::outs() << "(" << id << ") "
<< "Too Many Active Requests.\n";
} else {
@@ -27,12 +27,11 @@
#include <string>

namespace glow {
namespace runtime {

/// Callback signalling success/failure of loading a Module onto a device.
using ReadyCBTy = std::function<void(const Module *, runtime::ResultCode)>;
/// Callback signalling the result of running a function.
using ResultCBTy = std::function<void(
runtime::RunIdentifierTy, runtime::ResultCode, std::unique_ptr<Context>)>;

/// Map of Function name -> CompiledFunction, used when loading a network onto a
/// device.
using FunctionMapTy = std::map<std::string, CompiledFunction *>;
@@ -74,9 +73,9 @@ class DeviceManager {
/// functionName must match the name of a function already added.
/// Context should have all Placeholders allocated. resultCB will be called
/// with the Context results filled.
virtual runtime::RunIdentifierTy runFunction(std::string functionName,
std::unique_ptr<Context> ctx,
ResultCBTy resultCB) = 0;
virtual runtime::RunIdentifierTy
runFunction(std::string functionName, std::unique_ptr<Context> ctx,
runtime::ResultCBTy resultCB) = 0;

/// Stops execution and shuts down the Device.
virtual void stop(bool block = true) {}
@@ -96,6 +95,7 @@ class DeviceManager {
virtual bool isMemoryAvailable(uint64_t estimate) const = 0;
};

} // namespace runtime
} // namespace glow

#endif // GLOW_BACKENDS_DEVICEMANAGER_H
@@ -23,14 +23,14 @@
#include <atomic>

namespace glow {

namespace runtime {
class QueueBackedDeviceManager : public DeviceManager {
protected:
/// Thread which interfaces with the device.
ThreadPool workThread_;

/// Identifier for next run.
std::atomic<runtime::RunIdentifierTy> nextIdentifier_{1};
std::atomic<RunIdentifierTy> nextIdentifier_{1};

public:
QueueBackedDeviceManager(BackendKind backend, llvm::StringRef name)
@@ -64,11 +64,11 @@ class QueueBackedDeviceManager : public DeviceManager {
/// functionName must match the name of a function already added.
/// Context should have all Placeholders allocated. resultCB will be called
/// with the Context results filled.
runtime::RunIdentifierTy runFunction(std::string functionName,
std::unique_ptr<Context> ctx,
ResultCBTy callback) override {
RunIdentifierTy runFunction(std::string functionName,
std::unique_ptr<Context> ctx,
ResultCBTy callback) override {

runtime::RunIdentifierTy id = nextIdentifier_++;
RunIdentifierTy id = nextIdentifier_++;
workThread_.submit([this, id, functionName = std::move(functionName),
ctx = std::move(ctx),
callback = std::move(callback)]() mutable {
@@ -92,10 +92,10 @@ class QueueBackedDeviceManager : public DeviceManager {
virtual void evictNetworkImpl(std::string functionName) = 0;

/// Execute provided Function.
virtual void runFunctionImpl(runtime::RunIdentifierTy, std::string,
virtual void runFunctionImpl(RunIdentifierTy, std::string,
std::unique_ptr<Context>, ResultCBTy) = 0;
};

} // namespace runtime
} // namespace glow

#endif // GLOW_BACKENDS_QUEUEBACKEDDEVICEMANAGER_H
@@ -81,18 +81,6 @@ class NodeToFunctionMap {
GraphMemInfo getGraphMemInfo(Function *func) { return partitionCost_[func]; }
};

/// The struct contains all the created DAGNodes. This DAGNodeList owns all the
/// DAGNodes, which cannot outlive the DAGNodeList. In addition, the DAGNodes
/// can only refer to the DAGNodes from the same DAGNodeList, and they can use
/// the raw pointers to refer to each other since they are in the same
/// DAGNodeList.
struct DAGNodeList {
/// The root DAGNode pointer of each graph/function.
std::vector<std::unique_ptr<DAGNode>> roots;
/// The non-root DAGNode pointers.
std::vector<std::unique_ptr<DAGNode>> nodes;
};

/// Given a module, partitions each of the its functions into multiple ones
/// based on memory constraints and minimizes the communication cost.
class Partitioner {
@@ -38,16 +38,6 @@ enum class ExecutorKind {
/// a partitioned graph.
class Executor {
public:
/// Map of DeviceIDTy -> DeviceManager, used when selecting a DeviceManager
/// to use to execute a DAGNode.
using DeviceManagerMapTy =
std::map<DeviceIDTy, std::shared_ptr<DeviceManager>>;

/// Callback signalling the success/failure of running a DAG. The arguments
/// are the run identifier of the invocation, the execution result code and
/// a Context containing outputs.
using DoneCbTy = std::function<void(RunIdentifierTy, ResultCode,
std::unique_ptr<Context>)>;
/// Destructor.
virtual ~Executor() = default;

@@ -59,13 +49,13 @@ class Executor {
/// no postrequisites (i.e. the final results) in addition to the mappings
/// present in \p context.
virtual void run(const DAGNode *root, std::unique_ptr<Context> context,
RunIdentifierTy runId, DoneCbTy cb) = 0;
RunIdentifierTy runId, ResultCBTy cb) = 0;
};

/// Create an executor of kind \p kind that will call into the DeviceManager
/// instances provided in \deviceManagers. \returns a pointer to the
/// executor.
Executor *createExecutor(const Executor::DeviceManagerMapTy &deviceManagers,
Executor *createExecutor(const DeviceManagerMapTy &deviceManagers,
ExecutorKind executorKind = ExecutorKind::ThreadPool);

} // namespace runtime
@@ -29,7 +29,6 @@
namespace glow {
namespace runtime {

/// Callback signalling the result of running a function.
struct DeviceConfig {
std::string deviceName;
BackendKind backendKind;
@@ -69,7 +68,7 @@ class HostManager final {

/// A map of DeviceManagers by deviceID. An ordered map is used here to allow
/// a stable iteration order over devices.
std::map<DeviceIDTy, std::shared_ptr<DeviceManager>> devices_;
DeviceManagerMapTy devices_;

/// Executor class, this handles dispatching execution requests to the
/// appropriate device managers for an inference request.
@@ -24,15 +24,13 @@

namespace glow {
namespace runtime {
using DeviceIDtoManagerMapTy =
std::map<DeviceIDTy, std::shared_ptr<DeviceManager>>;

/// The Provisioner is responsible for assigning networks to an actual device.
/// It also compiles the networks before passing the compiled functions to the
/// device.
class Provisioner final {
public:
Provisioner(DeviceIDtoManagerMapTy &devices);
Provisioner(DeviceManagerMapTy &devices);

/// Walks \p networks and assigns each function to a DeviceManager in \p
/// devices. The Provisioner calls the addNetwork method for each
@@ -19,19 +19,24 @@
#include "glow/Backends/BackendUtils.h"
#include "glow/Graph/Graph.h"

#include <map>
#include <string>
#include <unordered_map>
#include <vector>

namespace glow {
namespace runtime {

class DeviceManager;
using DeviceIDTy = size_t;
using RunIdentifierTy = size_t;

/// Map of DeviceIDTy -> DeviceManager.
using DeviceManagerMapTy = std::map<DeviceIDTy, std::shared_ptr<DeviceManager>>;

/// Enum to communicate results when communicating with device at initialization
/// and runtime.
enum class ResultCode { Ready, Executed, Failed, Cancelled };
enum class ResultCode { Ready, Executed, Failed, Canceled };

/// Callback type used by HostManager and DeviceManager, used to pass results of
/// an inference request back to the caller.
@@ -67,6 +72,15 @@ struct DAGNode {
RuntimeBundle runtimeBundle;
};

/// This struct contains all the created DAGNodes from the Partitioner. The
/// contained DAGNodes can only refer to the DAGNodes from the same DAGNodeList.
struct DAGNodeList {
/// The root DAGNode pointer of each graph/function.
std::vector<std::unique_ptr<DAGNode>> roots;
/// The non-root DAGNode pointers.
std::vector<std::unique_ptr<DAGNode>> nodes;
};

} // namespace runtime
} // namespace glow
#endif // GLOW_RUNTIME_RUNTIMETYPES_H
@@ -21,9 +21,11 @@ using namespace glow;
using namespace glow::runtime;

namespace glow {
namespace runtime {
DeviceManager *createCPUDeviceManager(llvm::StringRef name) {
return new CPUDeviceManager(name);
}
} // namespace runtime
} // namespace glow

uint64_t CPUDeviceManager::getMaximumMemory() const { return maxMemoryBytes_; }
@@ -19,6 +19,7 @@
#include "glow/Backends/QueueBackedDeviceManager.h"

namespace glow {
namespace runtime {

/// A class controlling a single CPU thread of execution driving the JIT
/// backend. Many CPUFunctions may be added, but only one inference is executed
@@ -64,6 +65,7 @@ class CPUDeviceManager : public QueueBackedDeviceManager {
std::unique_ptr<Context> ctx, ResultCBTy cb) override;
};

} // namespace runtime
} // namespace glow

#endif // GLOW_BACKENDS_CPU_CPUDEVICEMANAGER_H
@@ -19,8 +19,10 @@
#include "llvm/Support/Casting.h"

using namespace glow;
using namespace glow::runtime;

namespace glow {
namespace runtime {
/// NOTE: Please add a declaration of a device-specific `create` method here
/// when you define a new DeviceManager.

@@ -49,6 +51,7 @@ DeviceManager *createOCLDeviceManager(llvm::StringRef name) {
GLOW_UNREACHABLE("Must compile with OpenCL support");
}
#endif
} // namespace runtime
} // namespace glow

DeviceManager *DeviceManager::createDeviceManager(BackendKind backendKind,
@@ -17,9 +17,8 @@

#include "llvm/Support/raw_ostream.h"

using namespace glow::runtime;

namespace glow {
namespace runtime {

DeviceManager *createInterpreterDeviceManager(llvm::StringRef name) {
return new InterpreterDeviceManager(name);
@@ -96,4 +95,5 @@ void InterpreterDeviceManager::runFunctionImpl(RunIdentifierTy id,
resultCB(id, ResultCode::Executed, std::move(ctx));
}

} // namespace runtime
} // namespace glow
@@ -19,6 +19,7 @@
#include "glow/Backends/QueueBackedDeviceManager.h"

namespace glow {
namespace runtime {

/// A class controlling a single "Interpreter Device", a thread of execution in
/// the IR-Interpreter. Many InterpreterFunctions may be added, but only one
@@ -64,6 +65,7 @@ class InterpreterDeviceManager : public QueueBackedDeviceManager {
std::unique_ptr<Context> ctx, ResultCBTy cb) override;
};

} // namespace runtime
} // namespace glow

#endif // GLOW_BACKENBDS_INTERPRETER_INTERPRETERDEVICEMANAGER_H
@@ -21,7 +21,7 @@
namespace glow {
namespace runtime {

Executor *createExecutor(const Executor::DeviceManagerMapTy &deviceManagers,
Executor *createExecutor(const DeviceManagerMapTy &deviceManagers,
ExecutorKind executorKind) {
switch (executorKind) {
case ExecutorKind::ThreadPool:
@@ -59,7 +59,7 @@ void InflightBarrier::wait() {

ExecutionState::ExecutionState(RunIdentifierTy id, const DAGNode *root,
std::unique_ptr<Context> resultContext,
DoneCbTy doneCb)
ResultCBTy doneCb)
: runId_(id), cb_(doneCb), resultCtx_(std::move(resultContext)),
inflightNodes_(0), resultCode_(ResultCode::Ready) {
// Create a queue for the breadth-first traversal through the graph.
@@ -213,7 +213,7 @@ ThreadPoolExecutor::~ThreadPoolExecutor() {

void ThreadPoolExecutor::run(const DAGNode *root,
std::unique_ptr<Context> context,
RunIdentifierTy runId, DoneCbTy cb) {
RunIdentifierTy runId, ResultCBTy cb) {
// If list of roots is empty, there is nothing to do. Give back the
// context so the caller can reuse it.
if (!root) {
@@ -381,7 +381,7 @@ void ThreadPoolExecutor::handleDeviceManagerResult(
if (noNodesInflight) {
// If there are no nodes inflight, that means all nodes are done. Call
// the callback and erase the state information.
DoneCbTy cb = executionState->getCallback();
ResultCBTy cb = executionState->getCallback();
cb(executionState->getRunId(), executionState->getResultCode(),
executionState->getUniqueResultContextPtr());

@@ -56,12 +56,10 @@ class InflightBarrier final {
/// by the runId).
class ExecutionState final {
public:
using DoneCbTy = Executor::DoneCbTy;

/// Constructor.
explicit ExecutionState(RunIdentifierTy id, const DAGNode *root,
std::unique_ptr<Context> resultContext,
DoneCbTy doneCb);
ResultCBTy doneCb);

/// Insert the placeholder-tensor pair (\p P, \p T) into the input context
/// for \p node. This should not be called at the same time as
@@ -99,7 +97,7 @@ class ExecutionState final {
Context *getRawResultContextPtr() const;

/// \returns the callback for this execution.
DoneCbTy getCallback() { return cb_; }
ResultCBTy getCallback() { return cb_; }

/// \returns the result code for the execution.
ResultCode getResultCode() const { return resultCode_; }
@@ -114,7 +112,7 @@ class ExecutionState final {
/// The run identifier for this execution of a DAG.
RunIdentifierTy runId_;
/// The callback that should be called when execution is done.
DoneCbTy cb_;
ResultCBTy cb_;
/// The Context object containing the results of the execution (i.e. the
/// outputs of the DAGNodes that have no children).
std::unique_ptr<Context> resultCtx_;
@@ -138,9 +136,6 @@ class ExecutionState final {
/// handle and process multiple concurrent execution runs.
class ThreadPoolExecutor final : public Executor {
public:
using DeviceManagerMapTy = Executor::DeviceManagerMapTy;
using DoneCbTy = Executor::DoneCbTy;

/// Constructor.
explicit ThreadPoolExecutor(const DeviceManagerMapTy &deviceManagers,
unsigned numWorkers = kNumWorkers)
@@ -152,7 +147,7 @@ class ThreadPoolExecutor final : public Executor {
/// See Executor::run. A particular invocation is specified completely by
/// the triple (roots, context, runId).
void run(const DAGNode *root, std::unique_ptr<Context> context,
RunIdentifierTy runId, DoneCbTy cb) override;
RunIdentifierTy runId, ResultCBTy cb) override;

private:
/// Propagate Placeholders from \p ctx into the final output Context for the
@@ -143,7 +143,7 @@ RunIdentifierTy HostManager::runNetwork(llvm::StringRef networkName,
return currentRun;
}
if (activeRequestCount_ >= activeRequestLimit_) {
callback(currentRun, ResultCode::Cancelled, std::move(context));
callback(currentRun, ResultCode::Canceled, std::move(context));
return currentRun;
}
activeRequestCount_++;
Oops, something went wrong.

0 comments on commit 1a21851

Please sign in to comment.