From 1f5e765d3f8452dedeba6a5fd6150b71223ebfb8 Mon Sep 17 00:00:00 2001 From: Gisle Dankel Date: Wed, 26 May 2021 17:06:49 -0700 Subject: [PATCH 1/2] Remove dependency on CuptiInterface from loggers Differential Revision: D28579642 fbshipit-source-id: 665fc347d9ad59f3492eb860c1dfe1872e122e7f --- libkineto/src/ActivityProfilerController.cpp | 5 ++--- libkineto/src/ActivityTrace.h | 10 +++------- libkineto/src/CudaDeviceProperties.cpp | 9 +++++++++ libkineto/src/CudaDeviceProperties.h | 2 ++ libkineto/src/output_json.cpp | 10 ++++------ libkineto/src/output_json.h | 7 +------ libkineto/test/ActivityProfilerTest.cpp | 6 +++--- 7 files changed, 24 insertions(+), 25 deletions(-) diff --git a/libkineto/src/ActivityProfilerController.cpp b/libkineto/src/ActivityProfilerController.cpp index 167cb8cf9..2845a0a7c 100644 --- a/libkineto/src/ActivityProfilerController.cpp +++ b/libkineto/src/ActivityProfilerController.cpp @@ -57,8 +57,7 @@ static std::unique_ptr makeLogger(const Config& config) { return loggerFactory()(config); } return std::make_unique( - config.activitiesLogFile(), - CuptiActivityInterface::singleton().smCount()); + config.activitiesLogFile()); } void ActivityProfilerController::profilerLoop() { @@ -138,7 +137,7 @@ std::unique_ptr ActivityProfilerController::stopTrace() auto logger = std::make_unique(profiler_->config()); profiler_->processTrace(*logger); profiler_->reset(); - return std::make_unique(std::move(logger), CuptiActivityInterface::singleton()); + return std::make_unique(std::move(logger)); } void ActivityProfilerController::addMetadata( diff --git a/libkineto/src/ActivityTrace.h b/libkineto/src/ActivityTrace.h index 37cc89525..3de844e8d 100644 --- a/libkineto/src/ActivityTrace.h +++ b/libkineto/src/ActivityTrace.h @@ -11,7 +11,6 @@ #include #include "ActivityTraceInterface.h" -#include "CuptiActivityInterface.h" #include "output_json.h" #include "output_membuf.h" @@ -19,23 +18,20 @@ namespace libkineto { class ActivityTrace : public ActivityTraceInterface { public: - ActivityTrace( - std::unique_ptr logger, - CuptiActivityInterface& cuptiActivities) - : logger_(std::move(logger)), cuptiActivities_(cuptiActivities) {} + explicit ActivityTrace(std::unique_ptr logger) + : logger_(std::move(logger)) {} const std::vector>* activities() override { return logger_->traceActivities(); }; void save(const std::string& path) override { - ChromeTraceLogger chrome_logger(path, cuptiActivities_.smCount()); + ChromeTraceLogger chrome_logger(path); logger_->log(chrome_logger); }; private: std::unique_ptr logger_; - CuptiActivityInterface& cuptiActivities_; }; } // namespace libkineto diff --git a/libkineto/src/CudaDeviceProperties.cpp b/libkineto/src/CudaDeviceProperties.cpp index 9b7ff2971..19591da93 100644 --- a/libkineto/src/CudaDeviceProperties.cpp +++ b/libkineto/src/CudaDeviceProperties.cpp @@ -23,13 +23,16 @@ static const std::vector createDeviceProps() { cudaError_t error_id = cudaGetDeviceCount(&device_count); // Return empty vector if error. if (error_id != cudaSuccess) { + LOG(ERROR) << "cudaGetDeviceCount failed with code " << error_id; return {}; } + VLOG(0) << "Device count is " << device_count; for (size_t i = 0; i < device_count; ++i) { cudaDeviceProp prop; error_id = cudaGetDeviceProperties(&prop, i); // Return empty vector if any device property fail to get. if (error_id != cudaSuccess) { + LOG(ERROR) << "cudaGetDeviceProperties failed with " << error_id; return {}; } props.push_back(prop); @@ -75,6 +78,12 @@ const std::string& devicePropertiesJson() { return devicePropsJson; } +int smCount(uint32_t deviceId) { + const std::vector &props = deviceProps(); + return deviceId >= props.size() ? 0 : + props[deviceId].multiProcessorCount; +} + float kernelOccupancy( uint32_t deviceId, uint16_t registersPerThread, diff --git a/libkineto/src/CudaDeviceProperties.h b/libkineto/src/CudaDeviceProperties.h index 751c56030..b731fde0c 100644 --- a/libkineto/src/CudaDeviceProperties.h +++ b/libkineto/src/CudaDeviceProperties.h @@ -12,6 +12,8 @@ namespace KINETO_NAMESPACE { +int smCount(uint32_t deviceId); + // Return estimated achieved occupancy for a kernel float kernelOccupancy( uint32_t deviceId, diff --git a/libkineto/src/output_json.cpp b/libkineto/src/output_json.cpp index 60cc2e760..77e184117 100644 --- a/libkineto/src/output_json.cpp +++ b/libkineto/src/output_json.cpp @@ -61,13 +61,10 @@ void ChromeTraceLogger::openTraceFile() { } } -ChromeTraceLogger::ChromeTraceLogger(const std::string& traceFileName, int smCount) +ChromeTraceLogger::ChromeTraceLogger(const std::string& traceFileName) : fileName_(traceFileName) { traceOf_.clear(std::ios_base::badbit); openTraceFile(); -#ifdef HAS_CUPTI - smCount_ = CuptiActivityInterface::singleton().smCount(); -#endif } static int64_t us(int64_t timestamp) { @@ -332,9 +329,10 @@ void ChromeTraceLogger::handleGpuActivity( constexpr int threads_per_warp = 32; float blocks_per_sm = -1.0; float warps_per_sm = -1.0; - if (smCount_) { + int sm_count = smCount(kernel->deviceId); + if (sm_count) { blocks_per_sm = - (kernel->gridX * kernel->gridY * kernel->gridZ) / (float) smCount_; + (kernel->gridX * kernel->gridY * kernel->gridZ) / (float) sm_count; warps_per_sm = blocks_per_sm * (kernel->blockX * kernel->blockY * kernel->blockZ) / threads_per_warp; diff --git a/libkineto/src/output_json.h b/libkineto/src/output_json.h index ba7c1e48d..a1436c1be 100644 --- a/libkineto/src/output_json.h +++ b/libkineto/src/output_json.h @@ -29,7 +29,7 @@ class Config; class ChromeTraceLogger : public libkineto::ActivityLogger { public: - explicit ChromeTraceLogger(const std::string& traceFileName, int smCount); + explicit ChromeTraceLogger(const std::string& traceFileName); // Note: the caller of these functions should handle concurrency // i.e., we these functions are not thread-safe @@ -82,11 +82,6 @@ class ChromeTraceLogger : public libkineto::ActivityLogger { std::string fileName_; std::ofstream traceOf_; - -#ifdef HAS_CUPTI - // Number of SMs on current device - int smCount_{0}; -#endif }; } // namespace KINETO_NAMESPACE diff --git a/libkineto/test/ActivityProfilerTest.cpp b/libkineto/test/ActivityProfilerTest.cpp index 13227ae7e..760904d0b 100644 --- a/libkineto/test/ActivityProfilerTest.cpp +++ b/libkineto/test/ActivityProfilerTest.cpp @@ -193,7 +193,7 @@ TEST(ActivityProfiler, AsyncTrace) { EXPECT_TRUE(success); EXPECT_FALSE(profiler.isActive()); - auto logger = std::make_unique(cfg.activitiesLogFile(), 10); + auto logger = std::make_unique(cfg.activitiesLogFile()); auto now = system_clock::now(); profiler.configure(cfg, now); profiler.setLogger(logger.get()); @@ -282,7 +282,7 @@ TEST_F(ActivityProfilerTest, SyncTrace) { profiler_->reset(); // Wrapper that allows iterating over the activities - ActivityTrace trace(std::move(logger), cuptiActivities_); + ActivityTrace trace(std::move(logger)); EXPECT_EQ(trace.activities()->size(), 9); std::map activityCounts; std::map resourceIds; @@ -362,7 +362,7 @@ TEST_F(ActivityProfilerTest, CorrelatedTimestampTest) { auto logger = std::make_unique(*cfg_); profiler.processTrace(*logger); - ActivityTrace trace(std::move(logger), cuptiActivities_); + ActivityTrace trace(std::move(logger)); std::map counts; for (auto& activity : *trace.activities()) { counts[activity->name()]++; From 807a1968185ad75a5cc0cabe4cc0d164570aa5c3 Mon Sep 17 00:00:00 2001 From: Gisle Dankel Date: Wed, 26 May 2021 17:07:03 -0700 Subject: [PATCH 2/2] Add support for multiple protocols in client API Summary: Add a way to add support for multiple protocols and associated logger objects. Reviewed By: ilia-cher Differential Revision: D28579660 fbshipit-source-id: 58ffad2206bf49bc7230521e2d2c6f9febff4b16 --- libkineto/src/AbstractConfig.h | 1 + libkineto/src/ActivityLoggerFactory.h | 64 ++++++++++++++++++++ libkineto/src/ActivityProfilerController.cpp | 25 +++++--- libkineto/src/ActivityProfilerController.h | 8 +-- libkineto/src/ActivityTrace.h | 27 ++++++--- libkineto/src/Config.cpp | 8 ++- libkineto/src/Config.h | 13 +++- libkineto/src/init.cpp | 3 +- libkineto/src/output_json.cpp | 10 ++- libkineto/src/output_json.h | 4 ++ libkineto/test/ActivityProfilerTest.cpp | 8 ++- 11 files changed, 141 insertions(+), 30 deletions(-) create mode 100644 libkineto/src/ActivityLoggerFactory.h diff --git a/libkineto/src/AbstractConfig.h b/libkineto/src/AbstractConfig.h index a4477d9ed..7b631ec39 100644 --- a/libkineto/src/AbstractConfig.h +++ b/libkineto/src/AbstractConfig.h @@ -52,6 +52,7 @@ class AbstractConfig { return timestamp_; } + // Source config string that this was parsed from const std::string& source() const { return source_; } diff --git a/libkineto/src/ActivityLoggerFactory.h b/libkineto/src/ActivityLoggerFactory.h new file mode 100644 index 000000000..c22d07c2c --- /dev/null +++ b/libkineto/src/ActivityLoggerFactory.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * All rights reserved. + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace KINETO_NAMESPACE { + +class ActivityLogger; + +class ActivityLoggerFactory { + + public: + using FactoryFunc = + std::function(const std::string& url)>; + + // Add logger factory for a protocol prefix + void addProtocol(const std::string& protocol, FactoryFunc f) { + factories_[tolower(protocol)] = f; + } + + // Create a logger, invoking the factory for the protocol specified in url + std::unique_ptr makeLogger(const std::string& url) const { + std::string protocol = extractProtocol(url); + auto it = factories_.find(tolower(protocol)); + if (it != factories_.end()) { + return it->second(stripProtocol(url)); + } + throw std::invalid_argument(fmt::format( + "No logger registered for the {} protocol prefix", + protocol)); + return nullptr; + } + + private: + static std::string tolower(std::string s) { + std::transform(s.begin(), s.end(), s.begin(), + [](unsigned char c) { return std::tolower(c); } + ); + return s; + } + + static std::string extractProtocol(std::string url) { + return url.substr(0, url.find("://")); + } + + static std::string stripProtocol(std::string url) { + size_t pos = url.find("://"); + return pos == url.npos ? url : url.substr(pos + 3); + } + + std::map factories_; +}; + +} // namespace KINETO_NAMESPACE diff --git a/libkineto/src/ActivityProfilerController.cpp b/libkineto/src/ActivityProfilerController.cpp index 2845a0a7c..64b049b74 100644 --- a/libkineto/src/ActivityProfilerController.cpp +++ b/libkineto/src/ActivityProfilerController.cpp @@ -10,6 +10,7 @@ #include #include +#include "ActivityLoggerFactory.h" #include "ActivityTrace.h" #include "CuptiActivityInterface.h" #include "ThreadUtil.h" @@ -39,25 +40,29 @@ ActivityProfilerController::~ActivityProfilerController() { VLOG(0) << "Stopped activity profiler"; } +static ActivityLoggerFactory initLoggerFactory() { + ActivityLoggerFactory factory; + factory.addProtocol("file", [](const std::string& url) { + return std::unique_ptr(new ChromeTraceLogger(url)); + }); + return factory; +} + static ActivityLoggerFactory& loggerFactory() { - static ActivityLoggerFactory factory{nullptr}; + static ActivityLoggerFactory factory = initLoggerFactory(); return factory; } -void ActivityProfilerController::setLoggerFactory( - const ActivityLoggerFactory& factory) { - loggerFactory() = factory; +void ActivityProfilerController::addLoggerFactory( + const std::string& protocol, ActivityLoggerFactory::FactoryFunc factory) { + loggerFactory().addProtocol(protocol, factory); } static std::unique_ptr makeLogger(const Config& config) { if (config.activitiesLogToMemory()) { return std::make_unique(config); } - if (loggerFactory()) { - return loggerFactory()(config); - } - return std::make_unique( - config.activitiesLogFile()); + return loggerFactory().makeLogger(config.activitiesLogUrl()); } void ActivityProfilerController::profilerLoop() { @@ -137,7 +142,7 @@ std::unique_ptr ActivityProfilerController::stopTrace() auto logger = std::make_unique(profiler_->config()); profiler_->processTrace(*logger); profiler_->reset(); - return std::make_unique(std::move(logger)); + return std::make_unique(std::move(logger), loggerFactory()); } void ActivityProfilerController::addMetadata( diff --git a/libkineto/src/ActivityProfilerController.h b/libkineto/src/ActivityProfilerController.h index d13b959a8..56eab1109 100644 --- a/libkineto/src/ActivityProfilerController.h +++ b/libkineto/src/ActivityProfilerController.h @@ -11,6 +11,7 @@ #include #include +#include "ActivityLoggerFactory.h" #include "ActivityProfiler.h" #include "ActivityProfilerInterface.h" #include "ActivityTraceInterface.h" @@ -20,9 +21,6 @@ namespace KINETO_NAMESPACE { class Config; -using ActivityLoggerFactory = - std::function(const Config&)>; - class ActivityProfilerController { public: explicit ActivityProfilerController(bool cpuOnly); @@ -32,7 +30,9 @@ class ActivityProfilerController { ~ActivityProfilerController(); - static void setLoggerFactory(const ActivityLoggerFactory& factory); + static void addLoggerFactory( + const std::string& protocol, + ActivityLoggerFactory::FactoryFunc factory); void scheduleTrace(const Config& config); diff --git a/libkineto/src/ActivityTrace.h b/libkineto/src/ActivityTrace.h index 3de844e8d..cd730c2e8 100644 --- a/libkineto/src/ActivityTrace.h +++ b/libkineto/src/ActivityTrace.h @@ -10,6 +10,7 @@ #include #include +#include "ActivityLoggerFactory.h" #include "ActivityTraceInterface.h" #include "output_json.h" #include "output_membuf.h" @@ -18,20 +19,32 @@ namespace libkineto { class ActivityTrace : public ActivityTraceInterface { public: - explicit ActivityTrace(std::unique_ptr logger) - : logger_(std::move(logger)) {} + ActivityTrace( + std::unique_ptr tmpLogger, + const ActivityLoggerFactory& factory) + : memLogger_(std::move(tmpLogger)), + loggerFactory_(factory) { + } const std::vector>* activities() override { - return logger_->traceActivities(); + return memLogger_->traceActivities(); }; - void save(const std::string& path) override { - ChromeTraceLogger chrome_logger(path); - logger_->log(chrome_logger); + void save(const std::string& url) override { + std::string prefix; + // if no protocol is specified, default to file + if (url.find("://") == url.npos) { + prefix = "file://"; + } + memLogger_->log(*loggerFactory_.makeLogger(prefix + url)); }; private: - std::unique_ptr logger_; + // Activities are logged into a buffer + std::unique_ptr memLogger_; + + // Alternative logger used by save() if protocol prefix is specified + const ActivityLoggerFactory& loggerFactory_; }; } // namespace libkineto diff --git a/libkineto/src/Config.cpp b/libkineto/src/Config.cpp index 9bd2a0c15..bb3927bc4 100644 --- a/libkineto/src/Config.cpp +++ b/libkineto/src/Config.cpp @@ -61,6 +61,7 @@ const string kHeartbeatMonitorPeriodKey = const string kActivitiesEnabledKey = "ACTIVITIES_ENABLED"; const string kActivityTypesKey = "ACTIVITY_TYPES"; const string kActivitiesLogFileKey = "ACTIVITIES_LOG_FILE"; +const string kActivitiesLogUrlKey = "ACTIVITIES_LOG_URL"; const string kActivitiesDurationKey = "ACTIVITIES_DURATION_SECS"; const string kActivitiesDurationMsecsKey = "ACTIVITIES_DURATION_MSECS"; const string kActivitiesIterationsKey = "ACTIVITIES_ITERATIONS"; @@ -118,16 +119,16 @@ const string kConfigFile = "/etc/libkineto.conf"; // Max devices supported on any system constexpr uint8_t kMaxDevices = 8; -static std::map>& +static std::map>& configFactories() { - static std::map> + static std::map> factories; return factories; } void Config::addConfigFactory( std::string name, - std::function factory) { + std::function factory) { configFactories()[name] = factory; } @@ -286,6 +287,7 @@ bool Config::handleOption(const std::string& name, std::string& val) { activityProfilerEnabled_ = toBool(val); } else if (name == kActivitiesLogFileKey) { activitiesLogFile_ = val; + activitiesLogUrl_ = fmt::format("file://{}", val); activitiesOnDemandTimestamp_ = timestamp(); } else if (name == kActivitiesMaxGpuBufferSizeKey) { activitiesMaxGpuBufferSize_ = toInt32(val) * 1024 * 1024; diff --git a/libkineto/src/Config.h b/libkineto/src/Config.h index 76a8d49d6..7cde942ec 100644 --- a/libkineto/src/Config.h +++ b/libkineto/src/Config.h @@ -53,6 +53,15 @@ class Config : public AbstractConfig { return activitiesLogFile_; } + // Log activitiy trace to this url + const std::string& activitiesLogUrl() const { + return activitiesLogUrl_; + } + + void setActivitiesLogUrl(const std::string& url) { + activitiesLogUrl_ = url; + } + bool activitiesLogToMemory() const { return activitiesLogToMemory_; } @@ -272,7 +281,7 @@ class Config : public AbstractConfig { static void addConfigFactory( std::string name, - std::function factory); + std::function factory); void print(std::ostream& s) const; @@ -338,6 +347,8 @@ class Config : public AbstractConfig { // The activity profiler settings are all on-demand std::string activitiesLogFile_; + std::string activitiesLogUrl_; + // Log activities to memory buffer bool activitiesLogToMemory_{false}; diff --git a/libkineto/src/init.cpp b/libkineto/src/init.cpp index d12e45794..8fe7928de 100644 --- a/libkineto/src/init.cpp +++ b/libkineto/src/init.cpp @@ -146,7 +146,8 @@ int InitializeInjection(void) { } void suppressLibkinetoLogMessages() { - SET_LOG_SEVERITY_LEVEL(ERROR); + //SET_LOG_SEVERITY_LEVEL(ERROR); + SET_LOG_VERBOSITY_LEVEL(2, {}); } } // extern C diff --git a/libkineto/src/output_json.cpp b/libkineto/src/output_json.cpp index 77e184117..f45de6e9e 100644 --- a/libkineto/src/output_json.cpp +++ b/libkineto/src/output_json.cpp @@ -30,6 +30,8 @@ using namespace libkineto; namespace KINETO_NAMESPACE { static constexpr int kSchemaVersion = 1; +static const std::string kDefaultLogFileFmt = + "/tmp/libkineto_activities_{}.json"; void ChromeTraceLogger::handleTraceStart( const std::unordered_map& metadata) { @@ -52,6 +54,10 @@ void ChromeTraceLogger::handleTraceStart( "traceEvents": [)JSON"; } +static std::string defaultFileName() { + return fmt::format(kDefaultLogFileFmt, processId()); +} + void ChromeTraceLogger::openTraceFile() { traceOf_.open(fileName_, std::ofstream::out | std::ofstream::trunc); if (!traceOf_) { @@ -61,8 +67,8 @@ void ChromeTraceLogger::openTraceFile() { } } -ChromeTraceLogger::ChromeTraceLogger(const std::string& traceFileName) - : fileName_(traceFileName) { +ChromeTraceLogger::ChromeTraceLogger(const std::string& traceFileName) { + fileName_ = traceFileName.empty() ? defaultFileName() : traceFileName; traceOf_.clear(std::ios_base::badbit); openTraceFile(); } diff --git a/libkineto/src/output_json.h b/libkineto/src/output_json.h index a1436c1be..a675321cd 100644 --- a/libkineto/src/output_json.h +++ b/libkineto/src/output_json.h @@ -68,6 +68,10 @@ class ChromeTraceLogger : public libkineto::ActivityLogger { std::unique_ptr buffers, int64_t endTime) override; + std::string traceFileName() const { + return fileName_; + } + private: #ifdef HAS_CUPTI diff --git a/libkineto/test/ActivityProfilerTest.cpp b/libkineto/test/ActivityProfilerTest.cpp index 760904d0b..4bab8d679 100644 --- a/libkineto/test/ActivityProfilerTest.cpp +++ b/libkineto/test/ActivityProfilerTest.cpp @@ -164,11 +164,15 @@ class ActivityProfilerTest : public ::testing::Test { profiler_ = std::make_unique( cuptiActivities_, /*cpu only*/ false); cfg_ = std::make_unique(); + loggerFactory.addProtocol("file", [](const std::string& url) { + return std::unique_ptr(new ChromeTraceLogger(url)); + }); } std::unique_ptr cfg_; MockCuptiActivities cuptiActivities_; std::unique_ptr profiler_; + ActivityLoggerFactory loggerFactory; }; @@ -282,7 +286,7 @@ TEST_F(ActivityProfilerTest, SyncTrace) { profiler_->reset(); // Wrapper that allows iterating over the activities - ActivityTrace trace(std::move(logger)); + ActivityTrace trace(std::move(logger), loggerFactory); EXPECT_EQ(trace.activities()->size(), 9); std::map activityCounts; std::map resourceIds; @@ -362,7 +366,7 @@ TEST_F(ActivityProfilerTest, CorrelatedTimestampTest) { auto logger = std::make_unique(*cfg_); profiler.processTrace(*logger); - ActivityTrace trace(std::move(logger)); + ActivityTrace trace(std::move(logger), loggerFactory); std::map counts; for (auto& activity : *trace.activities()) { counts[activity->name()]++;