Skip to content

Fix clock discrepancy #207

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Fix clock discrepancy #207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions libkineto/src/ActivityProfiler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,13 +497,13 @@ void ActivityProfiler::configure(
LOG(INFO) << "Enabling GPU tracing";
cupti_.setMaxBufferSize(config_->activitiesMaxGpuBufferSize());

time_point<high_resolution_clock> timestamp;
time_point<system_clock> timestamp;
if (VLOG_IS_ON(1)) {
timestamp = high_resolution_clock::now();
timestamp = system_clock::now();
}
cupti_.enableCuptiActivities(config_->selectedActivityTypes());
if (VLOG_IS_ON(1)) {
auto t2 = high_resolution_clock::now();
auto t2 = system_clock::now();
addOverheadSample(
setupOverhead_, duration_cast<microseconds>(t2 - timestamp).count());
}
Expand Down Expand Up @@ -538,13 +538,13 @@ void ActivityProfiler::stopTraceInternal(const time_point<system_clock>& now) {
}
#ifdef HAS_CUPTI
if (!cpuOnly_) {
time_point<high_resolution_clock> timestamp;
time_point<system_clock> timestamp;
if (VLOG_IS_ON(1)) {
timestamp = high_resolution_clock::now();
timestamp = system_clock::now();
}
cupti_.disableCuptiActivities(config_->selectedActivityTypes());
if (VLOG_IS_ON(1)) {
auto t2 = high_resolution_clock::now();
auto t2 = system_clock::now();
addOverheadSample(
setupOverhead_, duration_cast<microseconds>(t2 - timestamp).count());
}
Expand Down
6 changes: 3 additions & 3 deletions libkineto/src/CuptiActivityInterface.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,16 +153,16 @@ CuptiActivityInterface::activityBuffers() {

#ifdef HAS_CUPTI
VLOG(1) << "Flushing GPU activity buffers";
time_point<high_resolution_clock> t1;
time_point<system_clock> t1;
if (VLOG_IS_ON(1)) {
t1 = high_resolution_clock::now();
t1 = system_clock::now();
}
// Can't hold mutex_ during this call, since bufferCompleted
// will be called by libcupti and mutex_ is acquired there.
CUPTI_CALL(cuptiActivityFlushAll(CUPTI_ACTIVITY_FLAG_FLUSH_FORCED));
if (VLOG_IS_ON(1)) {
flushOverhead =
duration_cast<microseconds>(high_resolution_clock::now() - t1).count();
duration_cast<microseconds>(system_clock::now() - t1).count();
}
#endif
std::lock_guard<std::mutex> guard(mutex_);
Expand Down
10 changes: 5 additions & 5 deletions libkineto/src/EventProfiler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ void EventGroupSet::setEnabled(bool enabled) {

// Collect counter values for each counter in group set
void EventGroupSet::collectSample() {
auto timestamp = high_resolution_clock::now();
auto timestamp = system_clock::now();
for (int g = 0; g < set_.numEventGroups; g++) {
CUpti_EventGroup grp = set_.eventGroups[g];
for (const auto& id : cuptiEvents_.eventsInGroup(grp)) {
Expand All @@ -215,7 +215,7 @@ void EventGroupSet::collectSample() {
}

if (VLOG_IS_ON(1)) {
auto t2 = high_resolution_clock::now();
auto t2 = system_clock::now();
VLOG(1) << "Device " << cuptiEvents_.device() << " Sample (us): "
<< duration_cast<microseconds>(t2 - timestamp).count();
}
Expand Down Expand Up @@ -320,7 +320,7 @@ static unique_ptr<Config> alignAndValidateConfigs(
Config& base,
Config& onDemand) {
if (onDemand.eventProfilerOnDemandDuration().count() == 0 ||
high_resolution_clock::now() >
system_clock::now() >
(onDemand.eventProfilerOnDemandStartTime() +
onDemand.eventProfilerOnDemandDuration())) {
base.validate();
Expand Down Expand Up @@ -530,7 +530,7 @@ void EventProfiler::printAllSamples(ostream& s, CUdevice device) const {

void EventProfiler::enableNextCounterSet() {
if (sets_.size() > 1) {
auto t1 = high_resolution_clock::now();
auto t1 = system_clock::now();

VLOG(1) << "Disabling set " << curEnabledSet_;
sets_[curEnabledSet_].setEnabled(false);
Expand All @@ -539,7 +539,7 @@ void EventProfiler::enableNextCounterSet() {
sets_[curEnabledSet_].setEnabled(true);

if (VLOG_IS_ON(1)) {
auto t2 = high_resolution_clock::now();
auto t2 = system_clock::now();
VLOG(1) << "Switch (us): "
<< duration_cast<microseconds>(t2 - t1).count();
}
Expand Down
20 changes: 10 additions & 10 deletions libkineto/src/EventProfilerController.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,10 +273,10 @@ void EventProfilerController::profilerLoop() {

auto on_demand_config = std::make_unique<Config>();

time_point<high_resolution_clock> next_sample_time;
time_point<high_resolution_clock> next_report_time;
time_point<high_resolution_clock> next_on_demand_report_time;
time_point<high_resolution_clock> next_multiplex_time;
time_point<system_clock> next_sample_time;
time_point<system_clock> next_report_time;
time_point<system_clock> next_on_demand_report_time;
time_point<system_clock> next_multiplex_time;
bool reconfigure = true;
bool restart = true;
int report_count = 0;
Expand All @@ -296,7 +296,7 @@ void EventProfilerController::profilerLoop() {
reconfigure = true;
}

auto now = high_resolution_clock::now();
auto now = system_clock::now();
if (on_demand_config->eventProfilerOnDemandDuration().count() > 0 &&
now > (on_demand_config->eventProfilerOnDemandStartTime() +
on_demand_config->eventProfilerOnDemandDuration())) {
Expand All @@ -321,7 +321,7 @@ void EventProfilerController::profilerLoop() {
}

if (restart) {
now = high_resolution_clock::now();
now = system_clock::now();
next_sample_time = now + profiler_->samplePeriod();
next_report_time = now + profiler_->reportPeriod();
next_on_demand_report_time = now + profiler_->onDemandReportPeriod();
Expand All @@ -337,13 +337,13 @@ void EventProfilerController::profilerLoop() {
while (now < next_sample_time) {
/* sleep override */
std::this_thread::sleep_for(next_sample_time - now);
now = high_resolution_clock::now();
now = system_clock::now();
}
int sleep_time = duration_cast<milliseconds>(now - start_sleep).count();

auto start_sample = now;
profiler_->collectSample();
now = high_resolution_clock::now();
now = system_clock::now();
int sample_time = duration_cast<milliseconds>(now - start_sample).count();

next_sample_time += profiler_->samplePeriod();
Expand All @@ -366,7 +366,7 @@ void EventProfilerController::profilerLoop() {
next_on_demand_report_time += profiler_->onDemandReportPeriod();
}
profiler_->eraseReportedSamples();
now = high_resolution_clock::now();
now = system_clock::now();
int report_time = duration_cast<milliseconds>(now - start_report).count();

if (now > next_sample_time) {
Expand All @@ -380,7 +380,7 @@ void EventProfilerController::profilerLoop() {
profiler_->enableNextCounterSet();
next_multiplex_time += profiler_->multiplexPeriod();
}
now = high_resolution_clock::now();
now = system_clock::now();
int multiplex_time =
duration_cast<milliseconds>(now - start_multiplex).count();

Expand Down
4 changes: 2 additions & 2 deletions libkineto/test/EventProfilerTest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ TEST(PercentileTest, Normalize) {
TEST(EventTest, SumSamples) {
Event ev;
ev.instanceCount = 4;
auto t = high_resolution_clock::now();
auto t = system_clock::now();
ev.addSample(t, {1, 2, 3, 4});
ev.addSample(t, {10, 20, 30, 40});
ev.addSample(t, {100, 200, 300, 400});
Expand Down Expand Up @@ -94,7 +94,7 @@ TEST(EventTest, SumSamples) {
TEST(EventTest, Percentiles) {
Event ev;
ev.instanceCount = 4;
auto t = high_resolution_clock::now();
auto t = system_clock::now();
ev.addSample(t, {3, 2, 1, 4});
ev.addSample(t, {30, 20, 10, 40});
ev.addSample(t, {300, 200, 100, 400});
Expand Down