From df2d653940c5e1a45f4afe35a0635e4cfb42dad2 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Fri, 27 Nov 2020 15:42:38 +0100 Subject: [PATCH 01/60] fix the crash when IIOPMU is not initialized --- cpucounters.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpucounters.cpp b/cpucounters.cpp index 461bf285..b0d253c5 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -6597,7 +6597,7 @@ void PCM::programIIOCounters(IIOPMUCNTCTLRegister rawEvents[4], int IIOStack) else IIO_units.push_back(IIOStack); - for (int32 i = 0; (i < num_sockets) && MSR.size(); ++i) + for (int32 i = 0; (i < num_sockets) && MSR.size() && iioPMUs.size(); ++i) { uint32 refCore = socketRefCore[i]; TemporalThreadAffinity tempThreadAffinity(refCore); // speedup trick for Linux From e76f21dfaefb910a4ab390e38694d11d28f1e574 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 27 Nov 2020 17:20:03 +0100 Subject: [PATCH 02/60] pcm-raw: fix the name of the utility Change-Id: I13523637f7e012d5495c0e6de3871348fa8086d7 --- pcm-raw.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcm-raw.cpp b/pcm-raw.cpp index 492bdaef..5c2db2a4 100644 --- a/pcm-raw.cpp +++ b/pcm-raw.cpp @@ -372,7 +372,7 @@ int main(int argc, char* argv[]) #endif cerr << "\n"; - cerr << " Processor Counter Monitor: Core Monitoring Utility \n"; + cerr << " Processor Counter Monitor: Raw Event Monitoring Utility \n"; cerr << "\n"; double delay = -1.0; From bc851feba6317f5e46a0d1f67e4baa60f2627fca Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Mon, 30 Nov 2020 13:34:38 +0100 Subject: [PATCH 03/60] zero fixed counters if perf API is not used potential fix for https://github.com/opcm/pcm/issues/252 Change-Id: Ie038acaa5d1d8e34edc4bbe72605aea73d38c278 --- cpucounters.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpucounters.cpp b/cpucounters.cpp index b0d253c5..4639f295 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -2592,6 +2592,9 @@ PCM::ErrorCode PCM::programCoreCounters(const int i /* core */, ctrl_reg.fields.reserved1 = 0; } + MSR[i]->write(INST_RETIRED_ANY_ADDR, 0); + MSR[i]->write(CPU_CLK_UNHALTED_THREAD_ADDR, 0); + MSR[i]->write(CPU_CLK_UNHALTED_REF_ADDR, 0); MSR[i]->write(IA32_CR_FIXED_CTR_CTRL, ctrl_reg.value); } From e209a006b9aa7b73cfdbd62b21b5832eca507beb Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Tue, 1 Dec 2020 12:50:23 +0100 Subject: [PATCH 04/60] implement overflow correction logic for core events This is to protect against very unlikely occasional overflows when values reach 2^^48 for extremelely long collections in the own programming logic (not using perf API). Change-Id: I6b60a17b1100ea5c0d5d9364a788064c96578efe --- cpucounters.cpp | 36 ++++++++++++++++++++++++---------- cpucounters.h | 51 +++++++++++++++++++++++-------------------------- types.h | 3 +++ utils.h | 22 +++++++++++++++++++++ 4 files changed, 75 insertions(+), 37 deletions(-) diff --git a/cpucounters.cpp b/cpucounters.cpp index 4639f295..0c1e3528 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -2672,6 +2672,7 @@ PCM::ErrorCode PCM::programCoreCounters(const int i /* core */, value |= (1ULL << j); // enable all custom counters (if > 4) } + MSR[i]->write(IA32_PERF_GLOBAL_OVF_CTRL, value); MSR[i]->write(IA32_CR_PERF_GLOBAL_CTRL, value); } return PCM::Success; @@ -3655,6 +3656,7 @@ void BasicCounterState::readAndAggregate(std::shared_ptr msr) PCM * m = PCM::getInstance(); const int32 core_gen_counter_num_max = m->getMaxCustomCoreEvents(); + uint64 overflows = 0; const auto corruptedCountersMask = m->checkCustomCoreProgramming(msr); // reading core PMU counters @@ -3674,13 +3676,27 @@ void BasicCounterState::readAndAggregate(std::shared_ptr msr) else #endif { - msr->read(INST_RETIRED_ANY_ADDR, &cInstRetiredAny); - msr->read(CPU_CLK_UNHALTED_THREAD_ADDR, &cCpuClkUnhaltedThread); - msr->read(CPU_CLK_UNHALTED_REF_ADDR, &cCpuClkUnhaltedRef); - for (int i = 0; i < core_gen_counter_num_max; ++i) + uint64 overflows_after = 0; + + do { - msr->read(IA32_PMC0 + i, &cCustomEvents[i]); - } + msr->read(IA32_PERF_GLOBAL_STATUS, &overflows); // read overflows + // std::cerr << "Debug " << core_id << " IA32_PERF_GLOBAL_STATUS: " << overflows << std::endl; + + msr->read(INST_RETIRED_ANY_ADDR, &cInstRetiredAny); + msr->read(CPU_CLK_UNHALTED_THREAD_ADDR, &cCpuClkUnhaltedThread); + msr->read(CPU_CLK_UNHALTED_REF_ADDR, &cCpuClkUnhaltedRef); + for (int i = 0; i < core_gen_counter_num_max; ++i) + { + msr->read(IA32_PMC0 + i, &cCustomEvents[i]); + } + + msr->read(IA32_PERF_GLOBAL_STATUS, &overflows_after); // read overflows again + // std::cerr << "Debug " << core_id << " IA32_PERF_GLOBAL_STATUS: " << overflows << std::endl; + + } while (overflows != overflows_after); // repeat the reading if an overflow happened during the reading + + msr->write(IA32_PERF_GLOBAL_OVF_CTRL, overflows); // clear overflows } for (int i = 0; i < core_gen_counter_num_max; ++i) @@ -3713,12 +3729,12 @@ void BasicCounterState::readAndAggregate(std::shared_ptr msr) msr->read(MSR_SMI_COUNT, &cSMICount); - InstRetiredAny += m->extractCoreFixedCounterValue(cInstRetiredAny); - CpuClkUnhaltedThread += m->extractCoreFixedCounterValue(cCpuClkUnhaltedThread); - CpuClkUnhaltedRef += m->extractCoreFixedCounterValue(cCpuClkUnhaltedRef); + InstRetiredAny += checked_uint64(m->extractCoreFixedCounterValue(cInstRetiredAny), extract_bits(overflows, 32, 32)); + CpuClkUnhaltedThread += checked_uint64(m->extractCoreFixedCounterValue(cCpuClkUnhaltedThread), extract_bits(overflows, 33, 33)); + CpuClkUnhaltedRef += checked_uint64(m->extractCoreFixedCounterValue(cCpuClkUnhaltedRef), extract_bits(overflows, 34, 34)); for (int i = 0; i < core_gen_counter_num_max; ++i) { - Event(i) += m->extractCoreGenCounterValue(cCustomEvents[i]); + Event(i) += checked_uint64(m->extractCoreGenCounterValue(cCustomEvents[i]), extract_bits(overflows, i, i)); } //std::cout << "Scaling Factor " << m->L3ScalingFactor; cL3Occupancy = m->extractQOSMonitoring(cL3Occupancy); diff --git a/cpucounters.h b/cpucounters.h index b2a6827e..84f5a684 100644 --- a/cpucounters.h +++ b/cpucounters.h @@ -2031,35 +2031,35 @@ class BasicCounterState friend uint64 getSMICount(const CounterStateType & before, const CounterStateType & after); protected: - uint64 InstRetiredAny; - uint64 CpuClkUnhaltedThread; - uint64 CpuClkUnhaltedRef; + checked_uint64 InstRetiredAny; + checked_uint64 CpuClkUnhaltedThread; + checked_uint64 CpuClkUnhaltedRef; // dont put any additional fields between Event 0-Event 3 because getNumberOfCustomEvents assumes there are none union { - uint64 L3Miss; - uint64 Event0; - uint64 ArchLLCMiss; + checked_uint64 L3Miss; + checked_uint64 Event0; + checked_uint64 ArchLLCMiss; }; union { - uint64 L3UnsharedHit; - uint64 Event1; - uint64 ArchLLCRef; - uint64 SKLL3Hit; + checked_uint64 L3UnsharedHit; + checked_uint64 Event1; + checked_uint64 ArchLLCRef; + checked_uint64 SKLL3Hit; }; union { - uint64 L2HitM; - uint64 Event2; - uint64 SKLL2Miss; + checked_uint64 L2HitM; + checked_uint64 Event2; + checked_uint64 SKLL2Miss; }; union { - uint64 L2Hit; - uint64 Event3; + checked_uint64 L2Hit; + checked_uint64 Event3; }; - uint64 Event4, Event5, Event6, Event7; - uint64* getEventsPtr() { return &Event0; }; - const uint64* getEventsPtr() const { return &Event0; }; - uint64& Event(size_t i) { return getEventsPtr()[i]; }; - const uint64& Event(size_t i) const { return getEventsPtr()[i]; }; + checked_uint64 Event4, Event5, Event6, Event7; + checked_uint64* getEventsPtr() { return &Event0; }; + const checked_uint64* getEventsPtr() const { return &Event0; }; + checked_uint64& Event(size_t i) { return getEventsPtr()[i]; }; + const checked_uint64& Event(size_t i) const { return getEventsPtr()[i]; }; uint64 InvariantTSC; // invariant time stamp counter uint64 CStateResidency[PCM::MAX_C_STATE + 1]; int32 ThermalHeadroom; @@ -2070,9 +2070,6 @@ class BasicCounterState public: BasicCounterState() : - InstRetiredAny(0), - CpuClkUnhaltedThread(0), - CpuClkUnhaltedRef(0), InvariantTSC(0), ThermalHeadroom(PCM_INVALID_THERMAL_HEADROOM), L3Occupancy(0), @@ -2081,7 +2078,7 @@ class BasicCounterState SMICount(0) { memset(CStateResidency, 0, sizeof(CStateResidency)); - memset(getEventsPtr(), 0, sizeof(uint64) * PERF_MAX_CUSTOM_COUNTERS); + memset(getEventsPtr(), 0, sizeof(checked_uint64) * PERF_MAX_CUSTOM_COUNTERS); } virtual ~BasicCounterState() { } @@ -2810,7 +2807,7 @@ double getExecUsage(const CounterStateType & before, const CounterStateType & af template uint64 getInstructionsRetired(const CounterStateType & now) // instructions { - return now.InstRetiredAny; + return now.InstRetiredAny.getRawData_NoOverflowProtection(); } /*! \brief Computes the number core clock cycles when signal on a specific core is running (not halted) @@ -2862,7 +2859,7 @@ uint64 getRefCycles(const CounterStateType & before, const CounterStateType & af template uint64 getCycles(const CounterStateType & now) // clocks { - return now.CpuClkUnhaltedThread; + return now.CpuClkUnhaltedThread.getRawData_NoOverflowProtection(); } /*! \brief Computes average number of retired instructions per core cycle for the entire system combining instruction counts from logical cores to corresponding physical cores @@ -3192,7 +3189,7 @@ inline double getCoreCStateResidency(int state, const CounterStateType & before, template inline uint64 getCoreCStateResidency(int state, const CounterStateType& now) { - if (state == 0) return now.CpuClkUnhaltedRef; + if (state == 0) return now.CpuClkUnhaltedRef.getRawData_NoOverflowProtection(); return now.BasicCounterState::CStateResidency[state]; } diff --git a/types.h b/types.h index 16fc3a9e..b18964a5 100644 --- a/types.h +++ b/types.h @@ -56,6 +56,9 @@ typedef signed int int32; #define IA32_PERFEVTSEL2_ADDR (IA32_PERFEVTSEL0_ADDR + 2) #define IA32_PERFEVTSEL3_ADDR (IA32_PERFEVTSEL0_ADDR + 3) +constexpr auto IA32_PERF_GLOBAL_STATUS = 0x38E; +constexpr auto IA32_PERF_GLOBAL_OVF_CTRL = 0x390; + #define PERF_MAX_FIXED_COUNTERS (3) #define PERF_MAX_CUSTOM_COUNTERS (8) #define PERF_MAX_COUNTERS (PERF_MAX_FIXED_COUNTERS + PERF_MAX_CUSTOM_COUNTERS) diff --git a/utils.h b/utils.h index 5db372e7..c94cc4eb 100644 --- a/utils.h +++ b/utils.h @@ -150,7 +150,29 @@ class ThreadGroupTempAffinity }; #endif +class checked_uint64 // uint64 with checking for overflows when computing differences +{ + uint64 data; + uint64 overflows; +public: + checked_uint64() : data(0), overflows(0) {} + checked_uint64(const uint64 d, const uint64 o) : data(d), overflows(o) {} + const checked_uint64& operator += (const checked_uint64& o) + { + data += o.data; + overflows += o.overflows; + return *this; + } + uint64 operator - (const checked_uint64& o) const + { + // computing data - o.data + constexpr uint64 counter_width = 48; + return data + overflows * (1ULL << counter_width) - o.data; + } + + uint64 getRawData_NoOverflowProtection() const { return data; } +}; // a secure (but partial) alternative for sscanf // see example usage in pcm-core.cpp From bd5093d3721f642fd009f05a78f6ab55529b5155 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Tue, 1 Dec 2020 12:54:42 +0100 Subject: [PATCH 05/60] use full value if retrieved using perf API Change-Id: I39550c8845a9e22cf1e68f2c7f58cc12a81c1b2e --- cpucounters.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpucounters.cpp b/cpucounters.cpp index 0c1e3528..7d8993fe 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -353,6 +353,8 @@ uint64 extract_bits(uint64 myin, uint32 beg, uint32 end) uint64 PCM::extractCoreGenCounterValue(uint64 val) { + if (canUsePerf) return val; + if(core_gen_counter_width) return extract_bits(val, 0, core_gen_counter_width-1); @@ -361,6 +363,8 @@ uint64 PCM::extractCoreGenCounterValue(uint64 val) uint64 PCM::extractCoreFixedCounterValue(uint64 val) { + if (canUsePerf) return val; + if(core_fixed_counter_width) return extract_bits(val, 0, core_fixed_counter_width-1); From 7d000fc87cd8a0bcca7f07b3c866b84b917b0683 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 2 Dec 2020 10:53:37 +0100 Subject: [PATCH 06/60] fix pmu-query.py on Windows Change-Id: Ic5fd31fe1e7d6c7084a9afdcfc1d9b57cc5c0e6f --- pmu-query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmu-query.py b/pmu-query.py index 00276387..03937dc8 100755 --- a/pmu-query.py +++ b/pmu-query.py @@ -42,7 +42,7 @@ if platform.system() == 'CYGWIN_NT-6.1': p = subprocess.Popen(['./pcm-core.exe -c'],stdout=subprocess.PIPE,shell=True) elif platform.system() == 'Windows': - p = subprocess.Popen(['pcm-core.exe -c'],stdout=subprocess.PIPE,shell=True) + p = subprocess.Popen(['pcm-core.exe', '-c'],stdout=subprocess.PIPE,shell=True) else: p = subprocess.Popen(['./pcm-core.x -c'],stdout=subprocess.PIPE,shell=True) From 04fd9be217b8e64080d123660210567b8302b278 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 2 Dec 2020 11:47:54 +0100 Subject: [PATCH 07/60] don't ignore txt files Change-Id: I29fc0e5f0bc76ee2f30d9449b9f1fef091bf70f2 --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index ef99bb26..a847bc2e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ *.htm *.html *.dll -*.txt *.patch *.orig *.out From b919a897a27df6fd495c0346b84d3dd8200f5f76 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Sat, 5 Dec 2020 15:20:54 +0100 Subject: [PATCH 08/60] print the number of LLC slides per socket Change-Id: Ibb6cf541fe5c2e490c1845cda3511bad029aba00 --- cpucounters.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpucounters.cpp b/cpucounters.cpp index 7d8993fe..30025967 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -1408,6 +1408,7 @@ void PCM::printSystemTopology() const { std::cerr << "Physical cores per socket: " << num_phys_cores_per_socket << "\n"; } + std::cerr << "Last level cache slices per socket: " << getMaxNumOfCBoxes() << "\n"; std::cerr << "Core PMU (perfmon) version: " << perfmon_version << "\n"; std::cerr << "Number of core PMU generic (programmable) counters: " << core_gen_counter_num_max << "\n"; std::cerr << "Width of generic (programmable) counters: " << core_gen_counter_width << " bits\n"; From e4d2f26d3a960111ed6ac3a1510b395ad98bd4a7 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 9 Dec 2020 13:21:22 +0100 Subject: [PATCH 09/60] update the winpmem driver URL related to https://github.com/opcm/pcm/issues/248 Change-Id: I29f475bf3c6561e4a07fb6cc8ac43e14926a5422 --- mmio.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mmio.cpp b/mmio.cpp index 0a22ae76..a40f2413 100644 --- a/mmio.cpp +++ b/mmio.cpp @@ -53,7 +53,7 @@ class PCMPmem : public WinPmem { wcscat_s(driver_filename, MAX_PATH, L"\\winpmem_x64.sys"); if (GetFileAttributes(driver_filename) == INVALID_FILE_ATTRIBUTES) { - std::cerr << "ERROR: winpmem_x64.sys not found in current directory. Download it from https://github.com/google/rekall/raw/master/tools/pmem/resources/winpmem/winpmem_x64.sys .\n"; + std::cerr << "ERROR: winpmem_x64.sys not found in current directory. Download it from https://github.com/Velocidex/WinPmem/blob/master/kernel/binaries/winpmem_x64.sys .\n"; std::cerr << "ERROR: Memory bandwidth statistics will not be available.\n"; } break; @@ -61,7 +61,7 @@ class PCMPmem : public WinPmem { wcscat_s(driver_filename, MAX_PATH, L"\\winpmem_x86.sys"); if (GetFileAttributes(driver_filename) == INVALID_FILE_ATTRIBUTES) { - std::cerr << "ERROR: winpmem_x86.sys not found in current directory. Download it from https://github.com/google/rekall/raw/master/tools/pmem/resources/winpmem/winpmem_x86.sys .\n"; + std::cerr << "ERROR: winpmem_x86.sys not found in current directory. Download it from https://github.com/Velocidex/WinPmem/blob/master/kernel/binaries/winpmem_x86.sys .\n"; std::cerr << "ERROR: Memory bandwidth statistics will not be available.\n"; } break; @@ -204,4 +204,4 @@ MMIORange::~MMIORange() #endif -} // namespace pcm \ No newline at end of file +} // namespace pcm From ea0e5d3fa341530df7586d861f36f775cc270e52 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Thu, 17 Dec 2020 13:39:56 +0100 Subject: [PATCH 10/60] add (disabled) MSR write tracing on Linux --- msr.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/msr.cpp b/msr.cpp index e05e303f..916dbd89 100644 --- a/msr.cpp +++ b/msr.cpp @@ -39,6 +39,8 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #include #endif +#include + namespace pcm { #ifdef _MSC_VER @@ -236,6 +238,11 @@ MsrHandle::~MsrHandle() int32 MsrHandle::write(uint64 msr_number, uint64 value) { +#if 0 + static std::mutex m; + std::lock_guard g(m); + std::cout << "DEBUG: writing MSR 0x" << std::hex << msr_number << " value 0x" << value << " on cpu " << std::dec << cpu_id << std::endl; +#endif return ::pwrite(fd, (const void *)&value, sizeof(uint64), msr_number); } From 1a9ddf9bff259ed3869e78d74e7751a9b1c06557 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Thu, 17 Dec 2020 13:54:55 +0100 Subject: [PATCH 11/60] print info about perf uncore support --- cpucounters.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpucounters.cpp b/cpucounters.cpp index 30025967..65a10e14 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -4910,6 +4910,8 @@ bool PCM::useLinuxPerfForUncore() const static bool printed = false; bool secureBoot = isSecureBoot(); #ifdef PCM_USE_PERF + const auto imcIDs = enumeratePerfPMUs("imc", 100); + std::cout << "INFO: Linux perf interface to program uncore PMUs is " << (imcIDs.empty()?"NOT ":"") << "present\n"; const char * perf_env = std::getenv("PCM_USE_UNCORE_PERF"); if (perf_env != NULL && std::string(perf_env) == std::string("1")) { From dd6bfa57538b277fdbd47759dde640d00166c950 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Thu, 17 Dec 2020 14:15:27 +0100 Subject: [PATCH 12/60] execute the body of useLinuxPerfForUncore just once --- cpucounters.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/cpucounters.cpp b/cpucounters.cpp index 65a10e14..8106b566 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -4907,7 +4907,12 @@ bool PCM::isSecureBoot() const bool PCM::useLinuxPerfForUncore() const { - static bool printed = false; + static int use = -1; + if (use != -1) + { + return 1 == use; + } + use = 0; bool secureBoot = isSecureBoot(); #ifdef PCM_USE_PERF const auto imcIDs = enumeratePerfPMUs("imc", 100); @@ -4915,26 +4920,23 @@ bool PCM::useLinuxPerfForUncore() const const char * perf_env = std::getenv("PCM_USE_UNCORE_PERF"); if (perf_env != NULL && std::string(perf_env) == std::string("1")) { - if (!printed) std::cout << "INFO: using Linux perf interface to program uncore PMUs because env variable PCM_USE_UNCORE_PERF=1\n"; - printed = true; - return true; + std::cout << "INFO: using Linux perf interface to program uncore PMUs because env variable PCM_USE_UNCORE_PERF=1\n"; + use = 1; } if (secureBoot) { - if (!printed) std::cout << "Secure Boot detected. Using Linux perf for uncore PMU programming.\n"; - printed = true; - return true; + std::cout << "INFO: Secure Boot detected. Using Linux perf for uncore PMU programming.\n"; + use = 1; } else #endif { if (secureBoot) { - if (!printed) std::cerr << "ERROR: Secure Boot detected. Recompile PCM with -DPCM_USE_PERF or disable Secure Boot.\n"; - printed = true; + std::cerr << "ERROR: Secure Boot detected. Recompile PCM with -DPCM_USE_PERF or disable Secure Boot.\n"; } } - return false; + return 1 == use; } ServerPCICFGUncore::ServerPCICFGUncore(uint32 socket_, const PCM * pcm) : From 5a5e6a2bc738b7c32a22ac0241dbcd6e5bd725fa Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Mon, 21 Dec 2020 09:30:54 +0100 Subject: [PATCH 13/60] allow MBM on CLX and CPX --- cpucounters.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpucounters.cpp b/cpucounters.cpp index 30025967..8f118b68 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -676,7 +676,7 @@ bool PCM::L3CacheOccupancyMetricAvailable() const bool PCM::CoreLocalMemoryBWMetricAvailable() const { - if (cpu_model == SKX) return false; // SKZ4 errata + if (cpu_model == SKX && cpu_stepping < 5) return false; // SKZ4 errata PCM_CPUID_INFO cpuinfo; if (!(QOSMetricAvailable() && L3QOSMetricAvailable())) return false; @@ -686,7 +686,7 @@ bool PCM::CoreLocalMemoryBWMetricAvailable() const bool PCM::CoreRemoteMemoryBWMetricAvailable() const { - if (cpu_model == SKX) return false; // SKZ4 errata + if (cpu_model == SKX && cpu_stepping < 5) return false; // SKZ4 errata PCM_CPUID_INFO cpuinfo; if (!(QOSMetricAvailable() && L3QOSMetricAvailable())) return false; From f75a4f59b7495aec6a1d5c6e3faafc7f65fdfe78 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Mon, 21 Dec 2020 10:35:17 +0100 Subject: [PATCH 14/60] drop 'nice' Makefile target --- Makefile | 3 --- resctrl.h | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 resctrl.h diff --git a/Makefile b/Makefile index aa1bfad9..7c294b68 100644 --- a/Makefile +++ b/Makefile @@ -111,9 +111,6 @@ memoptest.x: memoptest.cpp dashboardtest.x: dashboardtest.cpp $(COMMON_OBJS) $(CXX) -o $@ $^ $(LIB) -nice: - uncrustify --replace -c ~/uncrustify.cfg *.cpp *.h WinMSRDriver/Win7/*.h WinMSRDriver/Win7/*.c WinMSRDriver/WinXP/*.h WinMSRDriver/WinXP/*.c PCM_Win/*.h PCM_Win/*.cpp - prefix=/usr install: all diff --git a/resctrl.h b/resctrl.h new file mode 100644 index 00000000..3b05cfc4 --- /dev/null +++ b/resctrl.h @@ -0,0 +1,20 @@ +/* +Copyright (c) 2020, Intel Corporation +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +// written by Roman Dementiev + + +/*! \file resctrl.h + \brief interface to MBM and CMT using Linux resctrl + */ + + \ No newline at end of file From 4d15bf80b4c668b5245909b7e4415e98a65288d9 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Mon, 21 Dec 2020 16:18:31 +0100 Subject: [PATCH 15/60] support resctl (draft) --- ENVVAR_README.md | 2 + Makefile | 2 +- cpucounters.cpp | 85 ++++++++++++++++++++++---- cpucounters.h | 14 ++++- resctrl.cpp | 154 +++++++++++++++++++++++++++++++++++++++++++++++ resctrl.h | 36 ++++++++++- utils.h | 6 ++ 7 files changed, 282 insertions(+), 17 deletions(-) create mode 100644 resctrl.cpp diff --git a/ENVVAR_README.md b/ENVVAR_README.md index 9abc2f01..a6153c18 100644 --- a/ENVVAR_README.md +++ b/ENVVAR_README.md @@ -3,3 +3,5 @@ `PCM_USE_UNCORE_PERF=1` : use Linux perf events API to program *uncore* PMUs (default is *not* to use it) `PCM_NO_RDT=1` : don't use RDT metrics for a better interoperation with pqos utility (https://github.com/intel/intel-cmt-cat) + +`PCM_USE_RESCTRL=1` : use Linux resctrl driver for RDT metrics diff --git a/Makefile b/Makefile index 7c294b68..cb86a1d6 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ CXX=c++ LIB= -lpthread -lc++ endif -COMMON_OBJS = msr.o cpucounters.o pci.o mmio.o client_bw.o utils.o topology.o dashboard.o debug.o threadpool.o +COMMON_OBJS = msr.o cpucounters.o pci.o mmio.o client_bw.o utils.o topology.o dashboard.o debug.o threadpool.o resctrl.o EXE_OBJS = $(EXE:.x=.o) OBJS = $(COMMON_OBJS) $(EXE_OBJS) diff --git a/cpucounters.cpp b/cpucounters.cpp index 8f118b68..46e817d3 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -650,7 +650,9 @@ bool PCM::isRDTDisabled() const bool PCM::QOSMetricAvailable() const { if (isRDTDisabled()) return false; - if (isSecureBoot()) return false; // TODO: use perf rdt driver +#ifndef __linux__ + if (isSecureBoot()) return false; +#endif PCM_CPUID_INFO cpuinfo; pcm_cpuid(0x7,0,cpuinfo); return (cpuinfo.reg.ebx & (1<<12))?true:false; @@ -659,7 +661,9 @@ bool PCM::QOSMetricAvailable() const bool PCM::L3QOSMetricAvailable() const { if (isRDTDisabled()) return false; - if (isSecureBoot()) return false; // TODO:: use perf rdt driver +#ifndef __linux__ + if (isSecureBoot()) return false; +#endif PCM_CPUID_INFO cpuinfo; pcm_cpuid(0xf,0,cpuinfo); return (cpuinfo.reg.edx & (1<<1))?true:false; @@ -703,10 +707,34 @@ unsigned PCM::getMaxRMID() const return maxRMID; } -void PCM::initRMID() +void PCM::initRDT() { if (!(QOSMetricAvailable() && L3QOSMetricAvailable())) return; +#ifdef __linux__ + auto env = std::getenv("PCM_USE_RESCTRL"); + if (env != nullptr && std::string(env) == std::string("1")) + { + std::cout << "INFO: using Linux resctrl driver for RDT metrics (L3OCC, LMB, RMB) because environment variable PCM_USE_RESCTRL=1\n"; + resctrl.init(); + useResctrl = true; + return; + } + if (resctrl.isMounted()) + { + std::cout << "INFO: using Linux resctrl driver for RDT metrics (L3OCC, LMB, RMB) because resctrl driver is mounted.\n"; + resctrl.init(); + useResctrl = true; + return; + } + if (isSecureBoot()) + { + std::cout << "INFO: using Linux resctrl driver for RDT metrics (L3OCC, LMB, RMB) because Secure Boot mode is enabled.\n"; + resctrl.init(); + useResctrl = true; + return; + } +#endif unsigned maxRMID; /* Calculate maximum number of RMID supported by socket */ maxRMID = getMaxRMID(); @@ -1901,6 +1929,10 @@ PCM::PCM() : allow_multiple_instances(false), programmed_pmu(false), joulesPerEnergyUnit(0), +#ifdef __linux__ + resctrl(*this), +#endif + useResctrl(false), disable_JKT_workaround(false), blocked(false), coreCStateMsr(NULL), @@ -1958,8 +1990,7 @@ PCM::PCM() : initUncoreObjects(); - // Initialize RMID to the cores for QOS monitoring - initRMID(); + initRDT(); readCPUMicrocodeLevel(); @@ -3353,11 +3384,18 @@ void PCM::resetPMU() std::cerr << " Zeroed PMU registers\n"; #endif } -void PCM::freeRMID() +void PCM::cleanupRDT() { if(!(QOSMetricAvailable() && L3QOSMetricAvailable())) { return; } +#ifdef __linux__ + if (useResctrl) + { + resctrl.cleanup(); + return; + } +#endif for(int32 core = 0; core < num_cores; core ++ ) { @@ -3419,7 +3457,7 @@ void PCM::cleanup() disableForceRTMAbortMode(); cleanupUncorePMUs(); - freeRMID(); + cleanupRDT(); #ifdef __linux__ if (needToRestoreNMIWatchdog) { @@ -3710,12 +3748,12 @@ void BasicCounterState::readAndAggregate(std::shared_ptr msr) } // std::cout << "DEBUG1: " << msr->getCoreId() << " " << cInstRetiredAny << " \n"; - if(m->L3CacheOccupancyMetricAvailable()) + if (m->L3CacheOccupancyMetricAvailable() && m->useResctrl == false) { msr->lock(); uint64 event = 1; m->initQOSevent(event, core_id); - msr->read(IA32_QM_CTR,&cL3Occupancy); + msr->read(IA32_QM_CTR, &cL3Occupancy); //std::cout << "readAndAggregate reading IA32_QM_CTR " << std::dec << cL3Occupancy << std::dec << "\n"; msr->unlock(); } @@ -3741,9 +3779,18 @@ void BasicCounterState::readAndAggregate(std::shared_ptr msr) { Event(i) += checked_uint64(m->extractCoreGenCounterValue(cCustomEvents[i]), extract_bits(overflows, i, i)); } - //std::cout << "Scaling Factor " << m->L3ScalingFactor; - cL3Occupancy = m->extractQOSMonitoring(cL3Occupancy); - L3Occupancy = (cL3Occupancy==PCM_INVALID_QOS_MONITORING_DATA)? PCM_INVALID_QOS_MONITORING_DATA : (uint64)((double)(cL3Occupancy * m->L3ScalingFactor) / 1024.0); +#ifdef __linux__ + if (m->useResctrl) + { + L3Occupancy = m->resctrl.getL3OCC(core_id) / 1024; + } + else +#endif + { + //std::cout << "Scaling Factor " << m->L3ScalingFactor; + cL3Occupancy = m->extractQOSMonitoring(cL3Occupancy); + L3Occupancy = (cL3Occupancy==PCM_INVALID_QOS_MONITORING_DATA)? PCM_INVALID_QOS_MONITORING_DATA : (uint64)((double)(cL3Occupancy * m->L3ScalingFactor) / 1024.0); + } for(int i=0; i <= int(PCM::MAX_C_STATE);++i) CStateResidency[i] += cCStateResidency[i]; ThermalHeadroom = extractThermalHeadroom(thermStatus); @@ -4153,6 +4200,20 @@ SystemCounterState PCM::getSystemCounterState() template void PCM::readAndAggregateMemoryBWCounters(const uint32 core, CounterStateType & result) { +#ifdef __linux__ + if (useResctrl) + { + if (CoreLocalMemoryBWMetricAvailable()) + { + result.MemoryBWLocal += resctrl.getMBL(core); + } + if (CoreRemoteMemoryBWMetricAvailable()) + { + result.MemoryBWTotal += resctrl.getMBT(core); + } + return; + } +#endif uint64 cMemoryBWLocal = 0; uint64 cMemoryBWTotal = 0; diff --git a/cpucounters.h b/cpucounters.h index 84f5a684..4fa223ec 100644 --- a/cpucounters.h +++ b/cpucounters.h @@ -67,6 +67,8 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #endif #endif +#include "resctrl.h" + namespace pcm { #ifdef _MSC_VER @@ -580,6 +582,10 @@ class PCM_API PCM std::vector > memory_bw_local; std::vector > memory_bw_total; +#ifdef __linux__ + Resctrl resctrl; +#endif + bool useResctrl; std::shared_ptr clientBW; std::shared_ptr clientImcReads; @@ -810,7 +816,7 @@ class PCM_API PCM bool PMUinUse(); void cleanupPMU(); - void freeRMID(); + void cleanupRDT(); bool decrementInstanceSemaphore(); // returns true if it was the last instance #ifdef __APPLE__ @@ -843,10 +849,12 @@ class PCM_API PCM * * \returns nothing */ - void initRMID(); + void initRDT(); /*! - * \brief initializes each core event MSR with an RMID for QOS event (L3 cache monitoring or memory bandwidth monitoring) + * \brief Initializes RDT * + * Initializes RDT infrastructure through resctrl Linux driver or direct MSR programming. + * For the latter: initializes each core event MSR with an RMID for QOS event (L3 cache monitoring or memory bandwidth monitoring) * \returns nothing */ void initQOSevent(const uint64 event, const int32 core); diff --git a/resctrl.cpp b/resctrl.cpp new file mode 100644 index 00000000..a0f3544b --- /dev/null +++ b/resctrl.cpp @@ -0,0 +1,154 @@ +/* +Copyright (c) 2020, Intel Corporation +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +// written by Roman Dementiev + +#ifdef __linux__ + +#include "resctrl.h" +#include "cpucounters.h" +#include +#include +#include +#include +#include + +namespace pcm +{ + bool Resctrl::isMounted() + { + struct stat st; + if (stat("/sys/fs/resctrl/mon_groups", &st) < 0) + { + return false; + } + return true; + } + void Resctrl::init() + { + if (isMounted() == false) + { + std::cerr << "ERROR: /sys/fs/resctrl is not mounted\n"; + std::cerr << "ERROR: RDT metrics (L3OCC,LMB,RMB) will not be available\n"; + std::cerr << "Mount it to make it work: mount -t resctrl resctrl /sys/fs/resctrl\n"; + return; + } + const auto numCores = pcm.getNumCores(); + fileLock.resize(numCores); + for (unsigned int c = 0; c < numCores; ++c) + { + fileLock[c] = std::unique_ptr(new std::mutex); + if (pcm.isCoreOnline(c)) + { + const auto C = std::to_string(c); + const auto dir = std::string(PCMPath) + C; + struct stat st; + if (stat(dir.c_str(), &st) < 0 && mkdir(dir.c_str(), 0700) < 0) + { + std::cerr << "INFO: can't create directory " << dir << " error: " << strerror(errno) << "\n"; + const auto containerDir = std::string("/pcm") + dir; + if (stat(containerDir.c_str(), &st) < 0 && mkdir(containerDir.c_str(), 0700) < 0) + { + std::cerr << "INFO: can't create directory " << containerDir << " error: " << strerror(errno) << "\n"; + std::cerr << "ERROR: RDT metrics (L3OCC,LMB,RMB) will not be available\n"; + break; + } + } + const auto cpus_listFilename = dir + "/cpus_list"; + writeSysFS(cpus_listFilename.c_str(), C, false); + auto openMetricFiles = [&dir, c] (PCM & pcm, const std::string & metric, FileMapType & fileMap) + { + auto getMetricFilename = [] (const std::string & dir, const uint64 s, const std::string & metric) + { + std::ostringstream ostr; + ostr << dir << "/mon_data/mon_L3_" << std::setfill('0') << std::setw(2) << s << "/" << metric; + return ostr.str(); + }; + for (uint64 s = 0; s < pcm.getNumSockets(); ++s) + { + auto occupancyPath = getMetricFilename(dir, s, metric); + auto f = tryOpen(occupancyPath.c_str(), "r"); + if (!f) + { + std::cerr << "ERROR: can't open "<< occupancyPath << "\n"; + } + else + { + fileMap[c].push_back(f); + } + } + }; + if (pcm.L3CacheOccupancyMetricAvailable()) + { + openMetricFiles(pcm, "llc_occupancy", L3OCC); + } + if (pcm.CoreLocalMemoryBWMetricAvailable()) + { + openMetricFiles(pcm, "mbm_local_bytes", MBL); + } + if (pcm.CoreRemoteMemoryBWMetricAvailable()) + { + openMetricFiles(pcm, "mbm_total_bytes", MBT); + } + } + } + } + void Resctrl::cleanup() + { + const auto numCores = pcm.getNumCores(); + for (unsigned int c = 0; c < numCores; ++c) + { + if (pcm.isCoreOnline(c)) + { + const auto dir = std::string(PCMPath) + std::to_string(c); + rmdir(dir.c_str()); + const auto containerDir = std::string("/pcm") + dir; + rmdir(containerDir.c_str()); + } + } + } + size_t Resctrl::getMetric(Resctrl::FileMapType & fileMap, int core) + { + if (core >= (int)fileLock.size()) return 0; + auto files = fileMap[core]; + size_t result = 0; + std::lock_guard _(*fileLock[core]); + for (auto f : files) + { + rewind(f); + char buffer[1024]; + if (NULL == fgets(buffer, 1024, f)) + { + std::cerr << "ERROR: Can not read from " << f << "\n"; + } + else + { + result += atoll(buffer); + } + } + return result; + } + size_t Resctrl::getL3OCC(int core) + { + return getMetric(L3OCC, core); + } + size_t Resctrl::getMBL(int core) + { + return getMetric(MBL, core); + } + size_t Resctrl::getMBT(int core) + { + return getMetric(MBT, core); + } +}; + + #endif // __linux__ \ No newline at end of file diff --git a/resctrl.h b/resctrl.h index 3b05cfc4..0289406a 100644 --- a/resctrl.h +++ b/resctrl.h @@ -12,9 +12,43 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND */ // written by Roman Dementiev +#pragma once /*! \file resctrl.h \brief interface to MBM and CMT using Linux resctrl */ - \ No newline at end of file +#ifdef __linux__ + +#include +#include +#include +#include +#include +#include + +namespace pcm +{ + class PCM; + + class Resctrl + { + PCM & pcm; + typedef std::unordered_map > FileMapType; + FileMapType L3OCC, MBL, MBT; + std::vector > fileLock; + Resctrl() = delete; + size_t getMetric(FileMapType & fileMap, int core); + static constexpr auto PCMPath = "/sys/fs/resctrl/mon_groups/pcm"; + public: + Resctrl(PCM & m) : pcm(m) {} + bool isMounted(); + void init(); + size_t getL3OCC(int core); + size_t getMBL(int core); + size_t getMBT(int core); + void cleanup(); + }; +}; + + #endif // __linux__ \ No newline at end of file diff --git a/utils.h b/utils.h index c94cc4eb..15c4f092 100644 --- a/utils.h +++ b/utils.h @@ -318,6 +318,12 @@ class MainLoop } }; +#ifdef __linux__ +FILE * tryOpen(const char * path, const char * mode); +std::string readSysFS(const char * path, bool silent); +bool writeSysFS(const char * path, const std::string & value, bool silent); +#endif + int calibratedSleep(const double delay, const char* sysCmd, const MainLoop& mainLoop, PCM* m); struct StackedBarItem { From 9a55ac87e621a9ca720896c2aba831920e0305ff Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Mon, 21 Dec 2020 16:34:24 +0100 Subject: [PATCH 16/60] open/close resctrl file for each read this is slow but no other way found to make it work with resctrl. rewind does not allow a new read. --- cpucounters.cpp | 4 ++-- daemon/daemon/Debug/objects.mk | 2 +- resctrl.cpp | 34 ++++++++-------------------------- resctrl.h | 3 +-- 4 files changed, 12 insertions(+), 31 deletions(-) diff --git a/cpucounters.cpp b/cpucounters.cpp index 46e817d3..d7aac227 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -4205,11 +4205,11 @@ void PCM::readAndAggregateMemoryBWCounters(const uint32 core, CounterStateType & { if (CoreLocalMemoryBWMetricAvailable()) { - result.MemoryBWLocal += resctrl.getMBL(core); + result.MemoryBWLocal += resctrl.getMBL(core) / (1024*1024); } if (CoreRemoteMemoryBWMetricAvailable()) { - result.MemoryBWTotal += resctrl.getMBT(core); + result.MemoryBWTotal += resctrl.getMBT(core) / (1024*1024); } return; } diff --git a/daemon/daemon/Debug/objects.mk b/daemon/daemon/Debug/objects.mk index d69bdddb..1f1b252b 100644 --- a/daemon/daemon/Debug/objects.mk +++ b/daemon/daemon/Debug/objects.mk @@ -2,7 +2,7 @@ # Automatically-generated file. Do not edit! ################################################################################ -USER_OBJS := ./utils.o ./pci.o ./msr.o ./mmio.o ./client_bw.o ./cpucounters.o ./topology.o ./debug.o ./threadpool.o +USER_OBJS := ./utils.o ./pci.o ./msr.o ./mmio.o ./client_bw.o ./cpucounters.o ./topology.o ./debug.o ./threadpool.o ./resctrl.o LIBS := -lpthread -lrt diff --git a/resctrl.cpp b/resctrl.cpp index a0f3544b..a13b39c0 100644 --- a/resctrl.cpp +++ b/resctrl.cpp @@ -43,10 +43,8 @@ namespace pcm return; } const auto numCores = pcm.getNumCores(); - fileLock.resize(numCores); for (unsigned int c = 0; c < numCores; ++c) { - fileLock[c] = std::unique_ptr(new std::mutex); if (pcm.isCoreOnline(c)) { const auto C = std::to_string(c); @@ -65,7 +63,7 @@ namespace pcm } const auto cpus_listFilename = dir + "/cpus_list"; writeSysFS(cpus_listFilename.c_str(), C, false); - auto openMetricFiles = [&dir, c] (PCM & pcm, const std::string & metric, FileMapType & fileMap) + auto generateMetricFiles = [&dir, c] (PCM & pcm, const std::string & metric, FileMapType & fileMap) { auto getMetricFilename = [] (const std::string & dir, const uint64 s, const std::string & metric) { @@ -75,29 +73,20 @@ namespace pcm }; for (uint64 s = 0; s < pcm.getNumSockets(); ++s) { - auto occupancyPath = getMetricFilename(dir, s, metric); - auto f = tryOpen(occupancyPath.c_str(), "r"); - if (!f) - { - std::cerr << "ERROR: can't open "<< occupancyPath << "\n"; - } - else - { - fileMap[c].push_back(f); - } + fileMap[c].push_back(getMetricFilename(dir, s, metric)); } }; if (pcm.L3CacheOccupancyMetricAvailable()) { - openMetricFiles(pcm, "llc_occupancy", L3OCC); + generateMetricFiles(pcm, "llc_occupancy", L3OCC); } if (pcm.CoreLocalMemoryBWMetricAvailable()) { - openMetricFiles(pcm, "mbm_local_bytes", MBL); + generateMetricFiles(pcm, "mbm_local_bytes", MBL); } if (pcm.CoreRemoteMemoryBWMetricAvailable()) { - openMetricFiles(pcm, "mbm_total_bytes", MBT); + generateMetricFiles(pcm, "mbm_total_bytes", MBT); } } } @@ -118,21 +107,14 @@ namespace pcm } size_t Resctrl::getMetric(Resctrl::FileMapType & fileMap, int core) { - if (core >= (int)fileLock.size()) return 0; auto files = fileMap[core]; size_t result = 0; - std::lock_guard _(*fileLock[core]); for (auto f : files) { - rewind(f); - char buffer[1024]; - if (NULL == fgets(buffer, 1024, f)) + const auto data = readSysFS(f.c_str(), false); + if (data.empty() == false) { - std::cerr << "ERROR: Can not read from " << f << "\n"; - } - else - { - result += atoll(buffer); + result += atoll(data.c_str()); } } return result; diff --git a/resctrl.h b/resctrl.h index 0289406a..d4a0c886 100644 --- a/resctrl.h +++ b/resctrl.h @@ -34,9 +34,8 @@ namespace pcm class Resctrl { PCM & pcm; - typedef std::unordered_map > FileMapType; + typedef std::unordered_map > FileMapType; FileMapType L3OCC, MBL, MBT; - std::vector > fileLock; Resctrl() = delete; size_t getMetric(FileMapType & fileMap, int core); static constexpr auto PCMPath = "/sys/fs/resctrl/mon_groups/pcm"; From 917724b6009c03132f8c35368dbb8fa900030ef4 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Mon, 21 Dec 2020 17:55:16 +0100 Subject: [PATCH 17/60] address g++10 warnings --- cpucounters.h | 5 ++++- pcm-sensor-server.cpp | 12 ++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cpucounters.h b/cpucounters.h index 4fa223ec..36f1cea8 100644 --- a/cpucounters.h +++ b/cpucounters.h @@ -2086,7 +2086,10 @@ class BasicCounterState SMICount(0) { memset(CStateResidency, 0, sizeof(CStateResidency)); - memset(getEventsPtr(), 0, sizeof(checked_uint64) * PERF_MAX_CUSTOM_COUNTERS); + Event0 = checked_uint64(); // default c-tor does not work for unions + Event1 = checked_uint64(); // default c-tor does not work for unions + Event2 = checked_uint64(); // default c-tor does not work for unions + Event3 = checked_uint64(); // default c-tor does not work for unions } virtual ~BasicCounterState() { } diff --git a/pcm-sensor-server.cpp b/pcm-sensor-server.cpp index a6d5451a..d6c47c44 100644 --- a/pcm-sensor-server.cpp +++ b/pcm-sensor-server.cpp @@ -288,21 +288,21 @@ class JSONPrinter : Visitor CoreCounterState const getCoreCounter( std::shared_ptr ag, uint32 tid ) const { CoreCounterState ccs; if ( nullptr == ag.get() ) - return std::move( ccs ); + return ccs; return std::move( ag->coreCounterStates()[tid] ); } SocketCounterState const getSocketCounter( std::shared_ptr ag, uint32 sid ) const { SocketCounterState socs; if ( nullptr == ag.get() ) - return std::move( socs ); + return socs; return std::move( ag->socketCounterStates()[sid] ); } SystemCounterState getSystemCounter( std::shared_ptr ag ) const { SystemCounterState sycs; if ( nullptr == ag.get() ) - return std::move( sycs ); + return sycs; return std::move( ag->systemCounterState() ); } @@ -548,21 +548,21 @@ class PrometheusPrinter : Visitor CoreCounterState const getCoreCounter( std::shared_ptr ag, uint32 tid ) const { CoreCounterState ccs; if ( nullptr == ag.get() ) - return std::move( ccs ); + return ccs; return std::move( ag->coreCounterStates()[tid] ); } SocketCounterState const getSocketCounter( std::shared_ptr ag, uint32 sid ) const { SocketCounterState socs; if ( nullptr == ag.get() ) - return std::move( socs ); + return socs; return std::move( ag->socketCounterStates()[sid] ); } SystemCounterState getSystemCounter( std::shared_ptr ag ) const { SystemCounterState sycs; if ( nullptr == ag.get() ) - return std::move( sycs ); + return sycs; return std::move( ag->systemCounterState() ); } From 059f6cb15d1cfad161fc66352b5dd6419e59a6fd Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Tue, 22 Dec 2020 10:14:27 +0100 Subject: [PATCH 18/60] refactor: get rid of unions for core event structure Change-Id: I16ea25463edab0d47b122d14e8d42be107b91bf3 --- cpucounters.cpp | 2 +- cpucounters.h | 68 ++++++++++++++++++------------------------------- 2 files changed, 26 insertions(+), 44 deletions(-) diff --git a/cpucounters.cpp b/cpucounters.cpp index 87b84670..5eb911cd 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -3777,7 +3777,7 @@ void BasicCounterState::readAndAggregate(std::shared_ptr msr) CpuClkUnhaltedRef += checked_uint64(m->extractCoreFixedCounterValue(cCpuClkUnhaltedRef), extract_bits(overflows, 34, 34)); for (int i = 0; i < core_gen_counter_num_max; ++i) { - Event(i) += checked_uint64(m->extractCoreGenCounterValue(cCustomEvents[i]), extract_bits(overflows, i, i)); + Event[i] += checked_uint64(m->extractCoreGenCounterValue(cCustomEvents[i]), extract_bits(overflows, i, i)); } #ifdef __linux__ if (m->useResctrl) diff --git a/cpucounters.h b/cpucounters.h index 36f1cea8..b3b25ed1 100644 --- a/cpucounters.h +++ b/cpucounters.h @@ -2042,32 +2042,18 @@ class BasicCounterState checked_uint64 InstRetiredAny; checked_uint64 CpuClkUnhaltedThread; checked_uint64 CpuClkUnhaltedRef; - // dont put any additional fields between Event 0-Event 3 because getNumberOfCustomEvents assumes there are none - union { - checked_uint64 L3Miss; - checked_uint64 Event0; - checked_uint64 ArchLLCMiss; + checked_uint64 Event[PERF_MAX_CUSTOM_COUNTERS]; + enum + { + L3MissPos = 0, + ArchLLCMissPos = 0, + L3UnsharedHitPos = 1, + ArchLLCRefPos = 1, + SKLL3HitPos = 1, + L2HitMPos = 2, + SKLL2MissPos = 2, + L2HitPos = 3 }; - union { - checked_uint64 L3UnsharedHit; - checked_uint64 Event1; - checked_uint64 ArchLLCRef; - checked_uint64 SKLL3Hit; - }; - union { - checked_uint64 L2HitM; - checked_uint64 Event2; - checked_uint64 SKLL2Miss; - }; - union { - checked_uint64 L2Hit; - checked_uint64 Event3; - }; - checked_uint64 Event4, Event5, Event6, Event7; - checked_uint64* getEventsPtr() { return &Event0; }; - const checked_uint64* getEventsPtr() const { return &Event0; }; - checked_uint64& Event(size_t i) { return getEventsPtr()[i]; }; - const checked_uint64& Event(size_t i) const { return getEventsPtr()[i]; }; uint64 InvariantTSC; // invariant time stamp counter uint64 CStateResidency[PCM::MAX_C_STATE + 1]; int32 ThermalHeadroom; @@ -2086,10 +2072,6 @@ class BasicCounterState SMICount(0) { memset(CStateResidency, 0, sizeof(CStateResidency)); - Event0 = checked_uint64(); // default c-tor does not work for unions - Event1 = checked_uint64(); // default c-tor does not work for unions - Event2 = checked_uint64(); // default c-tor does not work for unions - Event3 = checked_uint64(); // default c-tor does not work for unions } virtual ~BasicCounterState() { } @@ -2104,7 +2086,7 @@ class BasicCounterState CpuClkUnhaltedRef += o.CpuClkUnhaltedRef; for (int i = 0; i < PERF_MAX_CUSTOM_COUNTERS; ++i) { - Event(i) += o.Event(i); + Event[i] += o.Event[i]; } InvariantTSC += o.InvariantTSC; for (int i = 0; i <= (int)PCM::MAX_C_STATE; ++i) @@ -3018,7 +3000,7 @@ template uint64 getL3CacheMisses(const CounterStateType & before, const CounterStateType & after) { if (!PCM::getInstance()->isL3CacheMissesAvailable()) return 0; - return after.L3Miss - before.L3Miss; + return after.Event[BasicCounterState::L3MissPos] - before.Event[BasicCounterState::L3MissPos]; } /*! \brief Computes number of L2 cache misses @@ -3034,15 +3016,15 @@ uint64 getL2CacheMisses(const CounterStateType & before, const CounterStateType auto pcm = PCM::getInstance(); if (pcm->isL2CacheMissesAvailable() == false) return 0ULL; if (pcm->useSkylakeEvents()) { - return after.SKLL2Miss - before.SKLL2Miss; + return after.Event[BasicCounterState::SKLL2MissPos] - before.Event[BasicCounterState::SKLL2MissPos]; } if (pcm->isAtom() || pcm->getCPUModel() == PCM::KNL) { - return after.ArchLLCMiss - before.ArchLLCMiss; + return after.Event[BasicCounterState::ArchLLCMissPos] - before.Event[BasicCounterState::ArchLLCMissPos]; } - uint64 L3Miss = after.L3Miss - before.L3Miss; - uint64 L3UnsharedHit = after.L3UnsharedHit - before.L3UnsharedHit; - uint64 L2HitM = after.L2HitM - before.L2HitM; + uint64 L3Miss = after.Event[BasicCounterState::L3MissPos] - before.Event[BasicCounterState::L3MissPos]; + uint64 L3UnsharedHit = after.Event[BasicCounterState::L3UnsharedHitPos] - before.Event[BasicCounterState::L3UnsharedHitPos]; + uint64 L2HitM = after.Event[BasicCounterState::L2HitMPos] - before.Event[BasicCounterState::L2HitMPos]; return L2HitM + L3UnsharedHit + L3Miss; } @@ -3060,11 +3042,11 @@ uint64 getL2CacheHits(const CounterStateType & before, const CounterStateType & if (pcm->isL2CacheHitsAvailable() == false) return 0ULL; if (pcm->isAtom() || pcm->getCPUModel() == PCM::KNL) { - uint64 L2Miss = after.ArchLLCMiss - before.ArchLLCMiss; - uint64 L2Ref = after.ArchLLCRef - before.ArchLLCRef; + uint64 L2Miss = after.Event[BasicCounterState::ArchLLCMissPos] - before.Event[BasicCounterState::ArchLLCMissPos]; + uint64 L2Ref = after.Event[BasicCounterState::ArchLLCRefPos] - before.Event[BasicCounterState::ArchLLCRefPos]; return L2Ref - L2Miss; } - return after.L2Hit - before.L2Hit; + return after.Event[BasicCounterState::L2HitPos] - before.Event[BasicCounterState::L2HitPos]; } /*! \brief Computes L3 Cache Occupancy @@ -3112,7 +3094,7 @@ template uint64 getL3CacheHitsNoSnoop(const CounterStateType & before, const CounterStateType & after) { if (!PCM::getInstance()->isL3CacheHitsNoSnoopAvailable()) return 0; - return after.L3UnsharedHit - before.L3UnsharedHit; + return after.Event[BasicCounterState::L3UnsharedHitPos] - before.Event[BasicCounterState::L3UnsharedHitPos]; } /*! \brief Computes number of L3 cache hits where snooping in sibling L2 caches had to be done @@ -3127,9 +3109,9 @@ uint64 getL3CacheHitsSnoop(const CounterStateType & before, const CounterStateTy { if (!PCM::getInstance()->isL3CacheHitsSnoopAvailable()) return 0; if (PCM::getInstance()->useSkylakeEvents()) { - return after.SKLL3Hit - before.SKLL3Hit; + return after.Event[BasicCounterState::SKLL3HitPos] - before.Event[BasicCounterState::SKLL3HitPos]; } - return after.L2HitM - before.L2HitM; + return after.Event[BasicCounterState::L2HitMPos] - before.Event[BasicCounterState::L2HitMPos]; } @@ -3367,7 +3349,7 @@ uint64 getSMICount(const CounterStateType & before, const CounterStateType & aft template uint64 getNumberOfCustomEvents(int32 eventCounterNr, const CounterStateType & before, const CounterStateType & after) { - return after.Event(eventCounterNr) - before.Event(eventCounterNr); + return after.Event[eventCounterNr] - before.Event[eventCounterNr]; } /*! \brief Get estimation of QPI data traffic per incoming QPI link From 6af48d6b81af1e65b2044eba81d24db384c837f2 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Tue, 22 Dec 2020 12:26:10 +0100 Subject: [PATCH 19/60] resctrl: be more verbose in case of file read error --- resctrl.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/resctrl.cpp b/resctrl.cpp index a13b39c0..358e8e3a 100644 --- a/resctrl.cpp +++ b/resctrl.cpp @@ -116,6 +116,16 @@ namespace pcm { result += atoll(data.c_str()); } + else + { + static std::mutex lock; + std::lock_guard _(lock); + std::cerr << "Error reading " << f << ". Error: " << strerror(errno) << "\n"; + if (errno == 24) + { + std::cerr << "try executing 'ulimit -n 10000' to increase the limit on the number of open files.\n"; + } + } } return result; } From d677c2a17e1bc1e936bf1a12703b2d2456eb2108 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 23 Dec 2020 12:51:34 +0100 Subject: [PATCH 20/60] use fewer magic constants Change-Id: Ib8290fd7cdfd09112daad017d5aba9627e21b8ce --- cpucounters.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cpucounters.cpp b/cpucounters.cpp index 5eb911cd..784fa9e4 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -6762,9 +6762,9 @@ void PCM::programCbo(const uint64 * events, const uint32 opCode, const uint32 nc if((HASWELLX == cpu_model || BDX_DE == cpu_model || BDX == cpu_model || SKX == cpu_model) && llc_lookup_tid_filter != 0) *cboPMUs[i][cbo].filter[0] = llc_lookup_tid_filter; - PCM::program(cboPMUs[i][cbo], events, events + 4, UNC_PMON_UNIT_CTL_FRZ_EN); + PCM::program(cboPMUs[i][cbo], events, events + ServerUncoreCounterState::maxCounters, UNC_PMON_UNIT_CTL_FRZ_EN); - for (int c = 0; c < 4; ++c) + for (int c = 0; c < ServerUncoreCounterState::maxCounters; ++c) { *cboPMUs[i][cbo].counterValue[c] = 0; } @@ -6877,7 +6877,8 @@ void PCM::initLLCReadMissLatencyEvents(uint64 * events, uint32 & opCode) void PCM::programCbo() { - uint64 events[4] = {0, 0, 0, 0}; + uint64 events[ServerUncoreCounterState::maxCounters]; + std::fill(events, events + ServerUncoreCounterState::maxCounters, 0); uint32 opCode = 0; initLLCReadMissLatencyEvents(events, opCode); From 42a4f6b35106a87e37b42f825487535c097da910 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 23 Dec 2020 13:06:41 +0100 Subject: [PATCH 21/60] address a Klocwork warning Change-Id: I6267b360751024d7c12d35e766bc940a681a38b7 --- cpucounters.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpucounters.cpp b/cpucounters.cpp index 784fa9e4..3dd0ad61 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -4004,7 +4004,7 @@ PCM::ErrorCode PCM::program(const RawPMUConfigs& allPMUConfigs_) return PCM::UnknownError; } size_t c = 0; - for (; c < corePMUConfig.programmable.size() && c < (size_t)getMaxCustomCoreEvents() && c < PERF_MAX_COUNTERS; ++c) + for (; c < corePMUConfig.programmable.size() && c < (size_t)getMaxCustomCoreEvents() && c < PERF_MAX_CUSTOM_COUNTERS; ++c) { regs[c].value = corePMUConfig.programmable[c].first[0]; } From 17283bfdf9d44410471c9ebd31360ed8f2d5fb0e Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 23 Dec 2020 13:29:58 +0100 Subject: [PATCH 22/60] address Klocwork warnings (II) Change-Id: I6e7614be7e7962f6eff0514d2e2c4069ec8149f2 --- cpucounters.h | 2 ++ topology.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cpucounters.h b/cpucounters.h index b3b25ed1..2d27bd8d 100644 --- a/cpucounters.h +++ b/cpucounters.h @@ -532,6 +532,8 @@ class PCM_API PCM friend class Aggregator; friend class ServerPCICFGUncore; PCM(); // forbidden to call directly because it is a singleton + PCM(const PCM &) = delete; + PCM & operator = (const PCM &) = delete; int32 cpu_family; int32 cpu_model; diff --git a/topology.h b/topology.h index 3b73d9ce..ed91f820 100644 --- a/topology.h +++ b/topology.h @@ -286,6 +286,8 @@ class ClientUncore : public Uncore }; class Socket : public SystemObject { + Socket(const Socket &) = delete; + Socket & operator = (const Socket &) = delete; public: Socket( PCM* m, int32 apicID, int32 logicalID ); virtual ~Socket() { From ae61d39b4c8778479498cb19465aa26fcbfdca26 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 23 Dec 2020 13:59:36 +0100 Subject: [PATCH 23/60] add g++9 and g++10 to gitlab CI Change-Id: If4789fdbe7e30625f14b7b62fb09aa26cca6ea27 --- .gitlab-ci.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d62fac82..111c0790 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,6 +17,20 @@ build_linux:gcc5: - g++ --version - make -j +build_linux:gcc10: + image: "ubuntu:groovy" + stage: build + script: + - g++ --version + - make -j + +build_linux:gcc9: + image: "ubuntu:focal" + stage: build + script: + - g++ --version + - make -j + build_linux:gcc7: image: "ubuntu:bionic" stage: build From fb159f26fece168f2a6fc908b92a378375c3708d Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 23 Dec 2020 14:05:40 +0100 Subject: [PATCH 24/60] don't try to install g++-4.8 for groovy and focal Change-Id: If24208e76574fee14e6fb0cf6d1740255289e402 --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 111c0790..baaa0225 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,8 @@ build_linux:gcc5: build_linux:gcc10: image: "ubuntu:groovy" stage: build + before_script: + - apt-get update -qq && apt-get install -qq -y make g++ script: - g++ --version - make -j @@ -27,6 +29,8 @@ build_linux:gcc10: build_linux:gcc9: image: "ubuntu:focal" stage: build + before_script: + - apt-get update -qq && apt-get install -qq -y make g++ script: - g++ --version - make -j From 1a25c9c00a6323194d08daa614db6d909cbe7469 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 30 Dec 2020 14:58:25 +0100 Subject: [PATCH 25/60] don't hardcode the compiler Change-Id: Ief40371294675a94ab8e68a4d36e00bbfb234c22 --- Makefile | 2 +- daemon/client/Debug/makefile | 4 ++-- daemon/client/Debug/subdir.mk | 4 ++-- daemon/daemon/Debug/makefile | 6 +++--- daemon/daemon/Debug/subdir.mk | 4 ++-- daemon/daemon/test/Makefile | 7 +++---- 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index e5636ac5..cf4ba704 100644 --- a/Makefile +++ b/Makefile @@ -106,7 +106,7 @@ c_example_shlib.x: c_example.c libpcm.so @rm -f $*.d.tmp memoptest.x: memoptest.cpp - g++ -Wall -g -O0 -std=c++11 memoptest.cpp -o memoptest.x + $(CXX) -Wall -g -O0 -std=c++11 memoptest.cpp -o memoptest.x dashboardtest.x: dashboardtest.cpp $(COMMON_OBJS) $(CXX) -o $@ $^ $(LIB) diff --git a/daemon/client/Debug/makefile b/daemon/client/Debug/makefile index c4537f5c..9d5a0f6f 100644 --- a/daemon/client/Debug/makefile +++ b/daemon/client/Debug/makefile @@ -42,8 +42,8 @@ all: client # Tool invocations client: $(OBJS) $(USER_OBJS) @echo 'Building target: $@' - @echo 'Invoking: GCC C++ Linker' - g++ -o "client" $(OBJS) $(USER_OBJS) $(LIBS) + @echo 'Invoking: C++ Linker' + $(CXX) -o "client" $(OBJS) $(USER_OBJS) $(LIBS) @echo 'Finished building target: $@' @echo ' ' diff --git a/daemon/client/Debug/subdir.mk b/daemon/client/Debug/subdir.mk index 70765bc7..b35a03fd 100644 --- a/daemon/client/Debug/subdir.mk +++ b/daemon/client/Debug/subdir.mk @@ -19,8 +19,8 @@ CPP_DEPS += \ # Each subdirectory must supply rules for building sources it contributes %.o: ../%.cpp @echo 'Building file: $<' - @echo 'Invoking: GCC C++ Compiler' - g++ -O0 -g3 -Wall -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@:%.o=%.d)" -o "$@" "$<" + @echo 'Invoking: C++ Compiler' + $(CXX) -O0 -g3 -Wall -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@:%.o=%.d)" -o "$@" "$<" @echo 'Finished building: $<' @echo ' ' diff --git a/daemon/daemon/Debug/makefile b/daemon/daemon/Debug/makefile index a2e72f3c..ae0de046 100644 --- a/daemon/daemon/Debug/makefile +++ b/daemon/daemon/Debug/makefile @@ -47,8 +47,8 @@ endif # Tool invocations daemon: pre-build $(OBJS) @echo 'Building target: $@' - @echo 'Invoking: GCC C++ Linker' - g++ -o "daemon" $(OBJS) $(USER_OBJS) $(LIBS) + @echo 'Invoking: C++ Linker' + $(CXX) -o "daemon" $(OBJS) $(USER_OBJS) $(LIBS) @echo 'Finished building target: $@' @echo ' ' @@ -61,7 +61,7 @@ pre-build: $(USER_OBJS) %.o: ../../../%.cpp -@echo 'Build PCM' - -g++ ${CXXFLAGS} -c $< -o $@ -std=c++11 + -$(CXX) ${CXXFLAGS} -c $< -o $@ -std=c++11 -@echo ' ' .PHONY: all clean dependents diff --git a/daemon/daemon/Debug/subdir.mk b/daemon/daemon/Debug/subdir.mk index cbb149de..e5eccb4d 100644 --- a/daemon/daemon/Debug/subdir.mk +++ b/daemon/daemon/Debug/subdir.mk @@ -21,8 +21,8 @@ CPP_DEPS += \ # Each subdirectory must supply rules for building sources it contributes %.o: ../%.cpp @echo 'Building file: $<' - @echo 'Invoking: GCC C++ Compiler' - g++ -O0 -g3 -Wall -c -fmessage-length=0 -Wno-unknown-pragmas -std=c++11 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@:%.o=%.d)" -o "$@" "$<" + @echo 'Invoking: C++ Compiler' + $(CXX) -O0 -g3 -Wall -c -fmessage-length=0 -Wno-unknown-pragmas -std=c++11 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@:%.o=%.d)" -o "$@" "$<" @echo 'Finished building: $<' @echo ' ' diff --git a/daemon/daemon/test/Makefile b/daemon/daemon/test/Makefile index 22596a9e..ea66c6d7 100644 --- a/daemon/daemon/test/Makefile +++ b/daemon/daemon/test/Makefile @@ -1,4 +1,3 @@ -CC=g++ CFLAGS=-c -Wall -std=c++11 LDFLAGS= SOURCES=main.cpp @@ -9,11 +8,11 @@ RM=rm all: $(SOURCES) $(EXECUTABLE) $(EXECUTABLE): $(OBJECTS) - $(CC) $(LDFLAGS) $(OBJECTS) -o $@ + $(CXX) $(LDFLAGS) $(OBJECTS) -o $@ .cpp.o: - $(CC) $(CFLAGS) $< -o $@ + $(CXX) $(CFLAGS) $< -o $@ clean: -$(RM) $(OBJECTS) $(EXECUTABLE) - -@echo ' ' \ No newline at end of file + -@echo ' ' From eadbf9fe5db90ec5e17efa59ed01c222dc07f020 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 30 Dec 2020 15:17:36 +0100 Subject: [PATCH 26/60] daemon/client: fix error checking Change-Id: If19bdd6517e08390aa6f0c4e98bd182a016d88cc --- daemon/client/client.cpp | 2 +- daemon/daemon/daemon.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/client/client.cpp b/daemon/client/client.cpp index def97e2a..a9d956de 100644 --- a/daemon/client/client.cpp +++ b/daemon/client/client.cpp @@ -106,7 +106,7 @@ namespace PCMDaemon { { int sharedMemoryId; FILE *fp = fopen (shmIdLocation_.c_str(), "r"); - if (fp <= 0) + if (!fp) { std::cerr << "Failed to open to shared memory key location: " << shmIdLocation_ << "\n"; exit(EXIT_FAILURE); diff --git a/daemon/daemon/daemon.cpp b/daemon/daemon/daemon.cpp index 9e35048e..7e4dd22d 100644 --- a/daemon/daemon/daemon.cpp +++ b/daemon/daemon/daemon.cpp @@ -341,7 +341,7 @@ namespace PCMDaemon { //Store shm id in a file (shmIdLocation_) FILE *fp = fopen (shmIdLocation_.c_str(), "w"); - if (fp < 0) + if (!fp) { std::cerr << "Failed to create/write to shared memory key location: " << shmIdLocation_ << "\n"; exit(EXIT_FAILURE); From ec5c903e76ea32c2775cc452bfc5fcfe6ca75ee8 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 30 Dec 2020 15:29:58 +0100 Subject: [PATCH 27/60] address clang scan warning The warning: Value stored to 'atPos' is never read Change-Id: I8224eac6e5eff915373057c09771cce1722c028b --- pcm-sensor-server.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pcm-sensor-server.cpp b/pcm-sensor-server.cpp index d6c47c44..f446a4db 100644 --- a/pcm-sensor-server.cpp +++ b/pcm-sensor-server.cpp @@ -1522,7 +1522,7 @@ struct URL { authority = fullURL.substr( authorityPos+2, authorityEndPos - (authorityPos + 2) ); DBG( 3, "authority: '", authority, "'" ); - size_t atPos = authority.find( '@' ); + const size_t atPos = authority.find( '@' ); bool atFound = (atPos != std::string::npos); if ( atFound ) { if ( atPos == 0 ) @@ -1551,8 +1551,7 @@ struct URL { url.hasUser_ = true; // delete user/pass including the at authority.erase( 0, atPos+1 ); - } else - atPos = 0; + } // Instead of all the logic it is easier to work on substrings From d2bb303d5354299a333f5f9c3f323e06cc91c626 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 30 Dec 2020 15:34:01 +0100 Subject: [PATCH 28/60] address clang scan warning Value stored to 'EDC_Rd' is never read Change-Id: I704f5ff24028227e4d83baa98dffc7ee88f45afe --- pcm-memory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcm-memory.cpp b/pcm-memory.cpp index a6888ee0..af95781f 100644 --- a/pcm-memory.cpp +++ b/pcm-memory.cpp @@ -360,7 +360,7 @@ void display_bandwidth(PCM *m, memdata_t *md, const uint32 no_columns, const boo else { EDC_Rd = -1.0; - EDC_Rd = -1.0; + EDC_Wr = -1.0; } if (iMC_Rd >= 0.0 && iMC_Wr >= 0.0 && EDC_Rd >= 0.0 && EDC_Wr >= 0.0) From 8dde1fd729d5002c8e509437146824dd74c544ed Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 30 Dec 2020 16:29:09 +0100 Subject: [PATCH 29/60] pcm-memory: don't use KNL ID directly Change-Id: Iafffb135ce03c431f991ac8210b34cdcc03455fb --- pcm-memory.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pcm-memory.cpp b/pcm-memory.cpp index af95781f..a48479cc 100644 --- a/pcm-memory.cpp +++ b/pcm-memory.cpp @@ -526,7 +526,7 @@ void display_bandwidth_csv(PCM *m, memdata_t *md, uint64 /*elapsedTime*/, const << setw(8) << PMM_MM_Ratio(md, skt) << ','; }); } - if (m->getCPUModel() != PCM::KNL) + if (m->MCDRAMmemoryTrafficMetricsAvailable() == false) { if (md->PMM == false && md->PMMMixedMode == false) { @@ -661,9 +661,8 @@ void calculate_bandwidth(PCM *m, const ServerUncoreCounterState uncState1[], con return (float)(nEvents * 64 / 1000000.0 / (elapsedTime / 1000.0)); }; - switch (m->getCPUModel()) + if (m->MCDRAMmemoryTrafficMetricsAvailable()) { - case PCM::KNL: for (uint32 channel = 0; channel < max_edc_channels; ++channel) { if (skipInactiveChannels && getEDCCounter(channel, ServerPCICFGUncore::EventPosition::READ, uncState1[skt], uncState2[skt]) == 0.0 && getEDCCounter(channel, ServerPCICFGUncore::EventPosition::WRITE, uncState1[skt], uncState2[skt]) == 0.0) @@ -679,8 +678,9 @@ void calculate_bandwidth(PCM *m, const ServerUncoreCounterState uncState1[], con md.EDC_Rd_socket[skt] += md.EDC_Rd_socket_chan[skt][channel]; md.EDC_Wr_socket[skt] += md.EDC_Wr_socket_chan[skt][channel]; } - /* fall-through */ - default: + } + + { for (uint32 channel = 0; channel < max_imc_channels; ++channel) { uint64 reads = 0, writes = 0, pmmReads = 0, pmmWrites = 0, pmmMemoryModeCleanMisses = 0, pmmMemoryModeDirtyMisses = 0; From e888053e3348aad58bdb2c34888d3b88a3228027 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 30 Dec 2020 17:38:29 +0100 Subject: [PATCH 30/60] address clang scan warning Result of operation is garbage or undefined. 1st function call argument is an uninitialized value. Change-Id: Iabbf1e9ab5e467782ee6eecfb609b87b0251a8e6 --- pcm-memory.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pcm-memory.cpp b/pcm-memory.cpp index a48479cc..0b38e090 100644 --- a/pcm-memory.cpp +++ b/pcm-memory.cpp @@ -45,7 +45,7 @@ using namespace std; using namespace pcm; -const uint32 max_sockets = 256; +constexpr uint32 max_sockets = 256; uint32 max_imc_channels = ServerUncoreCounterState::maxChannels; const uint32 max_edc_channels = ServerUncoreCounterState::maxChannels; const uint32 max_imc_controllers = ServerUncoreCounterState::maxControllers; @@ -336,13 +336,13 @@ void display_bandwidth(PCM *m, memdata_t *md, const uint32 no_columns, const boo \r|-- DDR4 Channel Monitoring --||-- MCDRAM Channel Monitoring --|\n\ \r|---------------------------------------||---------------------------------------|\n\ \r"; - uint32 max_channels = max_imc_channels <= max_edc_channels ? max_edc_channels : max_imc_channels; + const uint32 max_channels = (std::max)(max_edc_channels, max_imc_channels); if (show_channel_output) { float iMC_Rd, iMC_Wr, EDC_Rd, EDC_Wr; for (uint64 channel = 0; channel < max_channels; ++channel) { - if (channel <= max_imc_channels) + if (channel < max_imc_channels) { iMC_Rd = md->iMC_Rd_socket_chan[skt][channel]; iMC_Wr = md->iMC_Wr_socket_chan[skt][channel]; @@ -352,7 +352,7 @@ void display_bandwidth(PCM *m, memdata_t *md, const uint32 no_columns, const boo iMC_Rd = -1.0; iMC_Wr = -1.0; } - if (channel <= max_edc_channels) + if (channel < max_edc_channels) { EDC_Rd = md->EDC_Rd_socket_chan[skt][channel]; EDC_Wr = md->EDC_Wr_socket_chan[skt][channel]; @@ -640,7 +640,7 @@ void calculate_bandwidth(PCM *m, const ServerUncoreCounterState uncState1[], con md.PMM = PMM; md.PMMMixedMode = PMMMixedMode; - for(uint32 skt = 0; skt < m->getNumSockets(); ++skt) + for(uint32 skt = 0; skt < max_sockets; ++skt) { md.iMC_Rd_socket[skt] = 0.0; md.iMC_Wr_socket[skt] = 0.0; @@ -654,6 +654,10 @@ void calculate_bandwidth(PCM *m, const ServerUncoreCounterState uncState1[], con { md.M2M_NM_read_hit_rate[skt][i] = 0.; } + } + + for(uint32 skt = 0; skt < m->getNumSockets(); ++skt) + { const uint32 numChannels1 = (uint32)m->getMCChannels(skt, 0); // number of channels in the first controller auto toBW = [&elapsedTime](const uint64 nEvents) From 0dbcf49400739708ddaccddb98223795eb86de0b Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 30 Dec 2020 17:58:31 +0100 Subject: [PATCH 31/60] address clang scan warning The right operand of '*' is a garbage value Change-Id: Ic84cb745ab33112250791248181f8cfd98738bcc --- c_example.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/c_example.c b/c_example.c index cae942e0..8b35d1f6 100644 --- a/c_example.c +++ b/c_example.c @@ -1,6 +1,7 @@ #include #include #include +#include int pcm_getcpu() { @@ -39,6 +40,12 @@ int main(int argc, const char *argv[]) { int i,a[100],b[100],c[100]; int lcore_id; + for (i=0; i < 100; ++i) + { + a[i] = rand(); + b[i] = rand(); + c[i] = rand(); + } #ifdef PCM_DYNAMIC_LIB void * handle = dlopen("libpcm.so", RTLD_LAZY); From fee35fc8438ce0a0f62c75109a3bf27142af4140 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 30 Dec 2020 09:51:36 -0800 Subject: [PATCH 32/60] add clang scan build Change-Id: Ib73974df560d320b2f3aac450811205f64b7baa1 --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index baaa0225..4eff7f38 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,6 +26,14 @@ build_linux:gcc10: - g++ --version - make -j +build_linux:clang_scan: + image: "ubuntu:groovy" + stage: build + before_script: + - apt-get update -qq && apt-get install -qq -y make clang clang-tools perl g++ + script: + - scan-build --status-bugs make -j + build_linux:gcc9: image: "ubuntu:focal" stage: build From 93ca579e9d90f2f691f1c6c7162f1b462a7f91a4 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 30 Dec 2020 19:41:02 +0100 Subject: [PATCH 33/60] address clang static analyzer warning Change-Id: Iddc12164b9e71a8c508ac49eee5eadbd8b058888 warning: Although the value stored to 'pid' is used in the enclosing expression, the value is never actually read from 'pid' [deadcode.DeadStores] --- pcm-sensor-server.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pcm-sensor-server.cpp b/pcm-sensor-server.cpp index f446a4db..811b184a 100644 --- a/pcm-sensor-server.cpp +++ b/pcm-sensor-server.cpp @@ -192,8 +192,7 @@ std::string read_ndctl_info( std::ofstream& logfile ) { exit(50); } std::stringstream ndctl; - int pid; - if ( (pid = fork()) == 0 ) { + if ( fork() == 0 ) { // child, writes to pipe, close read-end close( pipes[0] ); dup2( pipes[1], fileno(stdout) ); From 2f0ef08d15b223ce38f111fb7b91dcc729102a45 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 30 Dec 2020 20:55:38 +0100 Subject: [PATCH 34/60] address clang scan warnings cpucounters.cpp:291:19: warning: The result of the left shift is undefined due to shifting by '33', which is greater or equal to the width of type 'int' myll = (1 << (end + 1)) - 1; ^ ~~~~~~~~~ cpucounters.cpp:291:19: warning: The result of the left shift is undefined due to shifting by '63', which is greater or equal to the width of type 'int' myll = (1 << (end + 1)) - 1; ^ ~~~~~~~~~ Change-Id: I6dc6ea961641092d570d4e589f83d8fa2afe5a2c --- cpucounters.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpucounters.cpp b/cpucounters.cpp index 3dd0ad61..c98d41c1 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -281,6 +281,7 @@ PCM * PCM::getInstance() uint32 build_bit_ui(uint32 beg, uint32 end) { + assert(end <= 31); uint32 myll = 0; if (end == 31) { From 98452e030f12066ea3da2e548192af07b95f5a41 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 1 Jan 2021 12:29:12 +0100 Subject: [PATCH 35/60] address cppcheck warning Change-Id: I2f4b4827084625bd5b5757f9c48439b9c1f3594b --- realtime.cpp | 2 +- types.h | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/realtime.cpp b/realtime.cpp index 477d7aa6..7d5cafb3 100644 --- a/realtime.cpp +++ b/realtime.cpp @@ -79,7 +79,7 @@ template void measure(DS & ds, size_t repeat, size_t nelements) { SystemCounterState before_sstate, after_sstate; - double before_ts, after_ts; + double before_ts = 0.0, after_ts; // warm up std::find(ds.begin(), ds.end(), nelements); diff --git a/types.h b/types.h index b18964a5..073a8308 100644 --- a/types.h +++ b/types.h @@ -27,6 +27,7 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #include #include #include +#include #ifdef _MSC_VER #include @@ -1047,6 +1048,10 @@ struct MCFGRecord unsigned char startBusNumber; unsigned char endBusNumber; char reserved[4]; + MCFGRecord() + { + memset(this, 0, sizeof(MCFGRecord)); + } void print() { std::cout << "BaseAddress=" << (std::hex) << "0x" << baseAddress << " PCISegmentGroupNumber=0x" << PCISegmentGroupNumber << From 2b2438cea72a0e2b1be0b1087db8aee09d9e0f27 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 1 Jan 2021 12:34:20 +0100 Subject: [PATCH 36/60] add cppcheck to CI Change-Id: Id0e7aeeab2eb9cfa478982d93673b26ef305f54d --- .gitlab-ci.yml | 8 ++++++++ cppcheck.sh | 11 +++++++++++ 2 files changed, 19 insertions(+) create mode 100644 cppcheck.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4eff7f38..67ed5525 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,6 +34,14 @@ build_linux:clang_scan: script: - scan-build --status-bugs make -j +cppcheck: + image: "ubuntu:groovy" + stage: build + before_script: + - apt-get update -qq && apt-get install -qq -y cppcheck + script: + - sh cppcheck.sh . 10 + build_linux:gcc9: image: "ubuntu:focal" stage: build diff --git a/cppcheck.sh b/cppcheck.sh new file mode 100644 index 00000000..03758eb9 --- /dev/null +++ b/cppcheck.sh @@ -0,0 +1,11 @@ + +cppcheck $1 --force -j $2 2> cppcheck.out + +if [ -s cppcheck.out ] +then + cat cppcheck.out + exit 1 +fi + +echo No issues found + From e83ffb18fc261c2701df61753d593f6da80a246c Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 1 Jan 2021 13:29:47 +0100 Subject: [PATCH 37/60] cppcheck: ignore C# project PCM-Service_Win Change-Id: Iebce856060167bb3faf5d6afd92f95402ddb83a8 --- cppcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppcheck.sh b/cppcheck.sh index 03758eb9..11c40970 100644 --- a/cppcheck.sh +++ b/cppcheck.sh @@ -1,5 +1,5 @@ -cppcheck $1 --force -j $2 2> cppcheck.out +cppcheck $1 --force -iPCM-Service_Win -j $2 2> cppcheck.out if [ -s cppcheck.out ] then From e90946c51625d8c606c94053d3e3de19b4670ce3 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 1 Jan 2021 13:32:20 +0100 Subject: [PATCH 38/60] cppcheck: use 28 threads Change-Id: Ie96a2aff9a23c696c103d68e9924e273ab9b57c8 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 67ed5525..863639d8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,7 +40,7 @@ cppcheck: before_script: - apt-get update -qq && apt-get install -qq -y cppcheck script: - - sh cppcheck.sh . 10 + - sh cppcheck.sh . 28 build_linux:gcc9: image: "ubuntu:focal" From b7d1c511117c38e9c7bf834600abe04dfee0e430 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Fri, 1 Jan 2021 16:38:48 +0100 Subject: [PATCH 39/60] address cppcheck warnings Member variable ... is not initialized in the constructor. --- daemon/client/client.h | 2 +- daemon/daemon/common.h | 40 ++++++++++++++++++++-------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/daemon/client/client.h b/daemon/client/client.h index df8479b1..43c5ede0 100644 --- a/daemon/client/client.h +++ b/daemon/client/client.h @@ -38,7 +38,7 @@ namespace PCMDaemon { int pollIntervalMs_; std::string shmIdLocation_; bool shmAttached_; - PCMDaemon::SharedPCMState* sharedPCMState_; + PCMDaemon::SharedPCMState* sharedPCMState_ = nullptr; PCMDaemon::uint64 lastUpdatedClientTsc_; }; diff --git a/daemon/daemon/common.h b/daemon/daemon/common.h index b8b40e48..2554a257 100644 --- a/daemon/daemon/common.h +++ b/daemon/daemon/common.h @@ -62,30 +62,30 @@ namespace PCMDaemon { typedef struct PCMSystem PCMSystem; struct PCMCoreCounter { - uint64 coreId; - int32 socketId; - double instructionsPerCycle; - uint64 cycles; - uint64 instructionsRetired; - double execUsage; - double relativeFrequency; - double activeRelativeFrequency; - uint64 l3CacheMisses; - uint64 l3CacheReference; - uint64 l2CacheMisses; - double l3CacheHitRatio; - double l2CacheHitRatio; - double l3CacheMPI; - double l2CacheMPI; + uint64 coreId = 0; + int32 socketId = 0; + double instructionsPerCycle = 0.; + uint64 cycles = 0; + uint64 instructionsRetired = 0; + double execUsage = 0.; + double relativeFrequency = 0.; + double activeRelativeFrequency = 0.; + uint64 l3CacheMisses = 0; + uint64 l3CacheReference = 0; + uint64 l2CacheMisses = 0; + double l3CacheHitRatio = 0.; + double l2CacheHitRatio = 0.; + double l3CacheMPI = 0.; + double l2CacheMPI = 0.; bool l3CacheOccupancyAvailable; uint64 l3CacheOccupancy; bool localMemoryBWAvailable; uint64 localMemoryBW; bool remoteMemoryBWAvailable; uint64 remoteMemoryBW; - uint64 localMemoryAccesses; - uint64 remoteMemoryAccesses; - int32 thermalHeadroom; + uint64 localMemoryAccesses = 0; + uint64 remoteMemoryAccesses = 0; + int32 thermalHeadroom = 0; public: PCMCoreCounter() : @@ -131,7 +131,7 @@ namespace PCMDaemon { typedef struct PCMMemoryChannelCounter PCMMemoryChannelCounter; struct PCMMemorySocketCounter { - uint64 socketId; + uint64 socketId = 0; PCMMemoryChannelCounter channels[MEMORY_MAX_IMC_CHANNELS]; uint32 numOfChannels; float read; @@ -191,7 +191,7 @@ namespace PCMDaemon { typedef struct PCMQPILinkCounter PCMQPILinkCounter; struct PCMQPISocketCounter { - uint64 socketId; + uint64 socketId = 0; PCMQPILinkCounter links[QPI_MAX_LINKS]; uint64 total; From 839b5fcd4911f1cd841ed0bb9e1a4835f719052d Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Fri, 1 Jan 2021 16:45:24 +0100 Subject: [PATCH 40/60] cppcheck: fix silent strerror call Return value of function strerror() is not used --- pcm-sensor-server.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pcm-sensor-server.cpp b/pcm-sensor-server.cpp index 811b184a..bd58392f 100644 --- a/pcm-sensor-server.cpp +++ b/pcm-sensor-server.cpp @@ -848,7 +848,7 @@ class basic_socketbuf : public std::basic_streambuf { #endif bytesSent= ::send( socketFD_, (void*)outputBuffer_, bytesToSend, MSG_NOSIGNAL ); if ( -1 == bytesSent ) { - strerror( errno ); + std::cerr << strerror( errno ) << "\n"; return traits_type::eof(); } #if defined (USE_SSL) @@ -2651,7 +2651,7 @@ void HTTPServer::run() { socklen_t sa_len = sizeof( struct sockaddr_in ); int retval = ::accept( serverSocket_, (struct sockaddr*)&clientAddress, &sa_len ); if ( -1 == retval ) { - ::strerror( errno ); + std::cerr << ::strerror( errno ) << "\n"; continue; } clientSocketFD = retval; @@ -2661,7 +2661,7 @@ void HTTPServer::run() { ::memset( ipbuf, 0, 16 ); char const * resbuf = ::inet_ntop( AF_INET, &(clientAddress.sin_addr), ipbuf, INET_ADDRSTRLEN ); if ( nullptr == resbuf ) { - ::strerror( errno ); + std::cerr << ::strerror( errno ) << "\n"; ::close( clientSocketFD ); continue; } @@ -2738,7 +2738,7 @@ void HTTPSServer::run() { socklen_t sa_len = sizeof( struct sockaddr_in ); int retval = ::accept( serverSocket_, (struct sockaddr*)&clientAddress, &sa_len ); if ( -1 == retval ) { - ::strerror( errno ); + std::cerr << strerror( errno ) << "\n"; continue; } clientSocketFD = retval; @@ -2756,7 +2756,7 @@ void HTTPSServer::run() { memset( ipbuf, 0, 16 ); char const * resbuf = ::inet_ntop( AF_INET, &(clientAddress.sin_addr), ipbuf, INET_ADDRSTRLEN ); if ( nullptr == resbuf ) { - ::strerror( errno ); + std::cerr << strerror( errno ) << "\n"; ::close( clientSocketFD ); continue; } From 1a2c31de7d1a888dc82a13b2b8471b01ea3f5966 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Fri, 1 Jan 2021 17:16:09 +0100 Subject: [PATCH 41/60] address cppcheck warnings --- memoptest.cpp | 7 ++++--- readmem.cpp | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/memoptest.cpp b/memoptest.cpp index d3517e59..7d10d461 100644 --- a/memoptest.cpp +++ b/memoptest.cpp @@ -33,8 +33,8 @@ inline double my_timestamp() struct T { - int key[1]; - int data[3]; + int key[1] = { 0 }; + int data[3] = { 0, 0, 0 }; T() { } T(int a) { key[0] = a; } @@ -79,7 +79,8 @@ void stream_write_task(Y * p, Y * e, int value) template void read_intensive_task(Y * p, Y * e, int value) { - std::find(p, e, -1); + volatile const auto result = std::find(p, e, -1); + (result); } diff --git a/readmem.cpp b/readmem.cpp index aa20f458..7135a896 100644 --- a/readmem.cpp +++ b/readmem.cpp @@ -31,8 +31,8 @@ inline double my_timestamp() struct T { - int key[1]; - int data[15]; + int key[1] = { 0 }; + int data[15] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; T() { } T(int a) { key[0] = a; } @@ -46,7 +46,8 @@ struct T template void Memory_intensive_task(DS & ds) { - std::find(ds.begin(), ds.end(), ds.size()); + volatile const auto result = std::find(ds.begin(), ds.end(), ds.size()); + (result); } From a2085c0b084f694081a447c5f409bd5a86cac129 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Fri, 1 Jan 2021 17:24:11 +0100 Subject: [PATCH 42/60] address cppcheck warning Member variable 'HTTPSServer::sslCTX_' is not initialized in the constructor. --- pcm-sensor-server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcm-sensor-server.cpp b/pcm-sensor-server.cpp index bd58392f..a689d47c 100644 --- a/pcm-sensor-server.cpp +++ b/pcm-sensor-server.cpp @@ -2720,7 +2720,7 @@ class HTTPSServer : public HTTPServer { } private: - SSL_CTX* sslCTX_; + SSL_CTX* sslCTX_ = nullptr; std::string certificateFile_; std::string privateKeyFile_; }; From 23722ab9c717f71b303df2e581bc7faf2cd638df Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 1 Jan 2021 19:33:49 +0100 Subject: [PATCH 43/60] cppcheck: ignore realtime.cpp Change-Id: I2b5fda7f1ec80b6f9036b512f0d7c5db6338c208 --- cppcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppcheck.sh b/cppcheck.sh index 11c40970..fe458413 100644 --- a/cppcheck.sh +++ b/cppcheck.sh @@ -1,5 +1,5 @@ -cppcheck $1 --force -iPCM-Service_Win -j $2 2> cppcheck.out +cppcheck $1 --force -irealtime.cpp -iPCM-Service_Win -j $2 2> cppcheck.out if [ -s cppcheck.out ] then From 4335e09112ea7b54829de70f7c2171cbda9754c5 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 1 Jan 2021 20:34:48 +0100 Subject: [PATCH 44/60] cppcheck: enable 'warning' checks Change-Id: I5b87637b83e8201281dc2b36dabcb94773ea5f65 --- cppcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppcheck.sh b/cppcheck.sh index fe458413..eaa19859 100644 --- a/cppcheck.sh +++ b/cppcheck.sh @@ -1,5 +1,5 @@ -cppcheck $1 --force -irealtime.cpp -iPCM-Service_Win -j $2 2> cppcheck.out +cppcheck $1 --force --enable=warning -irealtime.cpp -iPCM-Service_Win -j $2 2> cppcheck.out if [ -s cppcheck.out ] then From a71785e198f5bd333d26d260765f2ba694be2793 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Sat, 2 Jan 2021 16:50:57 +0100 Subject: [PATCH 45/60] daemon: fix client build on older compilers Change-Id: I953afe4063fdbbed83272bf82e64b06ec7fe0f98 --- daemon/client/Debug/subdir.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/client/Debug/subdir.mk b/daemon/client/Debug/subdir.mk index b35a03fd..de0ec832 100644 --- a/daemon/client/Debug/subdir.mk +++ b/daemon/client/Debug/subdir.mk @@ -20,7 +20,7 @@ CPP_DEPS += \ %.o: ../%.cpp @echo 'Building file: $<' @echo 'Invoking: C++ Compiler' - $(CXX) -O0 -g3 -Wall -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@:%.o=%.d)" -o "$@" "$<" + $(CXX) -std=c++11 -O0 -g3 -Wall -c -fmessage-length=0 -MMD -MP -MF"$(@:%.o=%.d)" -MT"$(@:%.o=%.d)" -o "$@" "$<" @echo 'Finished building: $<' @echo ' ' From be8a9856d09abb7b66e6735ef5d1620291e73eb3 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Sat, 2 Jan 2021 17:20:42 +0100 Subject: [PATCH 46/60] cppcheck: fix a few warnings Change-Id: Ia5a1948dd1556687817118975e07312981472615 --- cppcheck.sh | 2 +- cpuasynchcounter.h | 4 ++-- daemon/daemon/daemon.h | 2 ++ memoptest.cpp | 3 +-- msrtest.cpp | 2 +- readmem.cpp | 3 +-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cppcheck.sh b/cppcheck.sh index eaa19859..27074d9f 100644 --- a/cppcheck.sh +++ b/cppcheck.sh @@ -1,5 +1,5 @@ -cppcheck $1 --force --enable=warning -irealtime.cpp -iPCM-Service_Win -j $2 2> cppcheck.out +cppcheck $1 --force --enable=warning --inline-suppr -irealtime.cpp -iPCM-Service_Win -j $2 2> cppcheck.out if [ -s cppcheck.out ] then diff --git a/cpuasynchcounter.h b/cpuasynchcounter.h index ca44ddca..c294330c 100644 --- a/cpuasynchcounter.h +++ b/cpuasynchcounter.h @@ -45,8 +45,8 @@ class AsynchronCounterState { friend void * UpdateCounters(void *); -// AsynchronCounterState(const& AsynchronCounterState); //unimplemeted -// const& AsynchronCounterState operator=(const& AsynchronCounterState); //unimplemented + AsynchronCounterState(const AsynchronCounterState &) = delete; + const AsynchronCounterState & operator = (const AsynchronCounterState &) = delete; public: AsynchronCounterState() diff --git a/daemon/daemon/daemon.h b/daemon/daemon/daemon.h index 9843ee6c..124c6d9e 100644 --- a/daemon/daemon/daemon.h +++ b/daemon/daemon/daemon.h @@ -32,6 +32,8 @@ namespace PCMDaemon { Daemon(int argc, char *argv[]); ~Daemon(); int run(); + Daemon (const Daemon &) = delete; + Daemon & operator = (const Daemon &) = delete; private: void setupPCM(); void checkAccessAndProgramPCM(); diff --git a/memoptest.cpp b/memoptest.cpp index 7d10d461..8015dae2 100644 --- a/memoptest.cpp +++ b/memoptest.cpp @@ -79,8 +79,7 @@ void stream_write_task(Y * p, Y * e, int value) template void read_intensive_task(Y * p, Y * e, int value) { - volatile const auto result = std::find(p, e, -1); - (result); + std::find(p, e, -1); // cppcheck-suppress ignoredReturnValue } diff --git a/msrtest.cpp b/msrtest.cpp index e4cb4555..0c50a2a7 100644 --- a/msrtest.cpp +++ b/msrtest.cpp @@ -30,7 +30,7 @@ int main() for (i = 0; i < NUM_CORES; ++i) { cpu_msr[i] = new MsrHandle(i); - assert(cpu_msr >= 0); + assert(cpu_msr[i]); FixedEventControlRegister ctrl_reg; res = cpu_msr[i]->read(IA32_CR_FIXED_CTR_CTRL, &ctrl_reg.value); diff --git a/readmem.cpp b/readmem.cpp index 7135a896..2f15caab 100644 --- a/readmem.cpp +++ b/readmem.cpp @@ -46,8 +46,7 @@ struct T template void Memory_intensive_task(DS & ds) { - volatile const auto result = std::find(ds.begin(), ds.end(), ds.size()); - (result); + std::find(ds.begin(), ds.end(), ds.size()); // cppcheck-suppress ignoredReturnValue } From c5f27ef96369e3b4e6d45b442f56dc470f6a9e6b Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Sat, 2 Jan 2021 17:33:05 +0100 Subject: [PATCH 47/60] cppcheck: include realtime.cpp again and fix a few warnings Change-Id: I848095e095af2773524798d94b64e9f4847396ca --- cppcheck.sh | 2 +- realtime.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cppcheck.sh b/cppcheck.sh index 27074d9f..263f41de 100644 --- a/cppcheck.sh +++ b/cppcheck.sh @@ -1,5 +1,5 @@ -cppcheck $1 --force --enable=warning --inline-suppr -irealtime.cpp -iPCM-Service_Win -j $2 2> cppcheck.out +cppcheck $1 --force --enable=warning --inline-suppr -iPCM-Service_Win -j $2 2> cppcheck.out if [ -s cppcheck.out ] then diff --git a/realtime.cpp b/realtime.cpp index 7d5cafb3..40c0dc86 100644 --- a/realtime.cpp +++ b/realtime.cpp @@ -82,7 +82,7 @@ void measure(DS & ds, size_t repeat, size_t nelements) double before_ts = 0.0, after_ts; // warm up - std::find(ds.begin(), ds.end(), nelements); + std::find(ds.begin(), ds.end(), nelements); // cppcheck-suppress ignoredReturnValue double before1_ts; #if 0 @@ -99,7 +99,7 @@ void measure(DS & ds, size_t repeat, size_t nelements) } #endif - for (int j = 0; j < repeat; ++j) std::find(ds.begin(), ds.end(), nelements); + for (int j = 0; j < repeat; ++j) std::find(ds.begin(), ds.end(), nelements); // cppcheck-suppress ignoredReturnValue // stop measuring after_sstate = getSystemCounterState(); @@ -143,8 +143,8 @@ typedef int T; struct T { - int key[1]; - int data[15]; + int key[1] = { 0 }; + int data[15] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };; T() { } T(int a) { key[0] = a; } From 2cbad233a252442b2d284b537fdead1aef49daca Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Mon, 4 Jan 2021 10:55:34 -0800 Subject: [PATCH 48/60] add compilation on windows with MSVC to CI --- .gitlab-ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 863639d8..03d47db4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,6 +34,15 @@ build_linux:clang_scan: script: - scan-build --status-bugs make -j +build_windows: + stage: build + before_script: + - 'call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat"' + script: + - 'msbuild pcm-all.sln /p:Configuration=Release;Platform=x64 /t:Clean,Build /m' + tags: + - windows + cppcheck: image: "ubuntu:groovy" stage: build From 381a753ffb51d35681c518d99dd194a276ab7ca6 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 15 Jan 2021 14:31:04 +0100 Subject: [PATCH 49/60] refactor SKL code path Change-Id: Ib91fbb3044e4146913bbb234665ba6e25fa746cc --- cpucounters.cpp | 20 ++++---------------- cpucounters.h | 42 ++++++++++++++++++++++++++---------------- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/cpucounters.cpp b/cpucounters.cpp index c98d41c1..9aa05306 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -847,11 +847,7 @@ void PCM::initCStateSupportTables() PCM_CSTATE_ARRAY(pkgCStateMsr, PCM_PARAM_PROTECT({0, 0, 0x60D, 0, 0, 0, 0x3F9, 0, 0, 0, 0}) ); case HASWELL_ULT: case BROADWELL: - case SKL: - case SKL_UY: - case KBL: - case KBL_1: - case ICL: + PCM_SKL_PATH_CASES case BROADWELL_XEON_E3: PCM_CSTATE_ARRAY(pkgCStateMsr, PCM_PARAM_PROTECT({0, 0, 0x60D, 0x3F8, 0, 0, 0x3F9, 0x3FA, 0x630, 0x631, 0x632}) ); @@ -891,11 +887,7 @@ void PCM::initCStateSupportTables() case CHERRYTRAIL: case APOLLO_LAKE: case DENVERTON: - case SKL_UY: - case SKL: - case KBL: - case KBL_1: - case ICL: + PCM_SKL_PATH_CASES PCM_CSTATE_ARRAY(coreCStateMsr, PCM_PARAM_PROTECT({0, 0, 0, 0x3FC, 0, 0, 0x3FD, 0x3FE, 0, 0, 0}) ); case KNL: PCM_CSTATE_ARRAY(coreCStateMsr, PCM_PARAM_PROTECT({0, 0, 0, 0, 0, 0, 0x3FF, 0, 0, 0, 0}) ); @@ -1504,9 +1496,7 @@ bool PCM::detectNominalFrequency() || cpu_model == AVOTON || cpu_model == APOLLO_LAKE || cpu_model == DENVERTON - || cpu_model == SKL - || cpu_model == KBL - || cpu_model == ICL + || useSKLPath() || cpu_model == KNL || cpu_model == SKX ) ? (100000000ULL) : (133333333ULL); @@ -2365,10 +2355,8 @@ PCM::ErrorCode PCM::program(const PCM::ProgramMode mode_, const void * parameter } else switch ( cpu_model ) { - case SKL: + PCM_SKL_PATH_CASES case SKX: - case KBL: - case ICL: assert(useSkylakeEvents()); coreEventDesc[0].event_number = SKL_MEM_LOAD_RETIRED_L3_MISS_EVTNR; coreEventDesc[0].umask_value = SKL_MEM_LOAD_RETIRED_L3_MISS_UMASK; diff --git a/cpucounters.h b/cpucounters.h index 2d27bd8d..346e1e4e 100644 --- a/cpucounters.h +++ b/cpucounters.h @@ -1262,6 +1262,26 @@ class PCM_API PCM END_OF_MODEL_LIST = 0x0ffff }; +#define PCM_SKL_PATH_CASES \ + case PCM::SKL_UY: \ + case PCM::KBL: \ + case PCM::KBL_1: \ + case PCM::CML: \ + case PCM::ICL: \ + case PCM::SKL: + +private: + bool useSKLPath() const + { + switch (cpu_model) + { + PCM_SKL_PATH_CASES + return true; + } + return false; + } +public: + //! \brief Reads CPU model id //! \return CPU model ID uint32 getCPUModel() const { return (uint32)cpu_model; } @@ -1426,9 +1446,7 @@ class PCM_API PCM case BROADWELL: case BDX_DE: case BDX: - case SKL: - case KBL: - case ICL: + PCM_SKL_PATH_CASES case SKX: return 4; case KNL: @@ -1680,9 +1698,7 @@ class PCM_API PCM || cpu_model == PCM::BDX_DE || cpu_model == PCM::BDX || cpu_model == PCM::KNL - || cpu_model == PCM::SKL - || cpu_model == PCM::KBL - || cpu_model == PCM::ICL + || useSKLPath() || cpu_model == PCM::SKX ); } @@ -1764,9 +1780,7 @@ class PCM_API PCM || cpu_model == PCM::IVY_BRIDGE || cpu_model == PCM::HASWELL || cpu_model == PCM::BROADWELL - || cpu_model == PCM::SKL - || cpu_model == PCM::KBL - || cpu_model == PCM::ICL + || useSKLPath() ); } @@ -1783,7 +1797,7 @@ class PCM_API PCM cpu_model == PCM::HASWELLX || cpu_model == PCM::BDX || cpu_model == PCM::SKX - || cpu_model == PCM::SKL + || useSKLPath() ); } @@ -1872,9 +1886,7 @@ class PCM_API PCM bool useSkylakeEvents() const { - return PCM::SKL == cpu_model - || PCM::KBL == cpu_model - || PCM::ICL == cpu_model + return useSKLPath() || PCM::SKX == cpu_model ; } @@ -1885,9 +1897,7 @@ class PCM_API PCM || cpu_model == IVY_BRIDGE || cpu_model == HASWELL || cpu_model == BROADWELL - || cpu_model == SKL - || cpu_model == KBL - || cpu_model == ICL + || useSKLPath() ; } From 6079d1156793b866336249b33cc86f47402cbc6b Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 15 Jan 2021 14:35:34 +0100 Subject: [PATCH 50/60] ICL is 5-way Change-Id: I022c8e346a7df960f15322f6cbdb7b3b502960bd --- cpucounters.h | 1 + 1 file changed, 1 insertion(+) diff --git a/cpucounters.h b/cpucounters.h index 346e1e4e..8f796eeb 100644 --- a/cpucounters.h +++ b/cpucounters.h @@ -1428,6 +1428,7 @@ class PCM_API PCM //! \return max number of instructions per cycle uint32 getMaxIPC() const { + if (ICL == cpu_model) return 5; switch (cpu_model) { case DENVERTON: From 804bc3d1b9e4e7ece92ba9ddfbc85d6a07b727f9 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 15 Jan 2021 14:51:24 +0100 Subject: [PATCH 51/60] properly detect codename for Skylake U/Y, Whiskey and Comet Lake Change-Id: Ic5be8f08fca941e2e9f0b7189b5da5a3ad94f4a0 --- cpucounters.cpp | 14 ++++++++++---- cpucounters.h | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/cpucounters.cpp b/cpucounters.cpp index 9aa05306..d376369b 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -2055,7 +2055,7 @@ bool PCM::isSocketOnline(int32 socket_id) const return socketRefCore[socket_id] != -1; } -bool PCM::isCPUModelSupported(int model_) +bool PCM::isCPUModelSupported(const int model_) { return ( model_ == NEHALEM_EP || model_ == NEHALEM_EX @@ -2074,7 +2074,10 @@ bool PCM::isCPUModelSupported(int model_) || model_ == BROADWELL || model_ == KNL || model_ == SKL + || model_ == SKL_UY || model_ == KBL + || model_ == KBL_1 + || model_ == CML || model_ == ICL || model_ == SKX ); @@ -2086,9 +2089,6 @@ bool PCM::checkModel() if (cpu_model == ATOM_2) cpu_model = ATOM; if (cpu_model == HASWELL_ULT || cpu_model == HASWELL_2) cpu_model = HASWELL; if (cpu_model == BROADWELL_XEON_E3) cpu_model = BROADWELL; - if (cpu_model == SKL_UY) cpu_model = SKL; - if (cpu_model == KBL_1) cpu_model = KBL; - if (cpu_model == CML) cpu_model = KBL; if(!isCPUModelSupported((int)cpu_model)) { @@ -3254,8 +3254,14 @@ const char * PCM::getUArchCodename(const int32 cpu_model_param) const return "Broadwell"; case SKL: return "Skylake"; + case SKL_UY: + return "Skylake U/Y"; case KBL: return "Kabylake"; + case KBL_1: + return "Kabylake/Whiskey Lake"; + case CML: + return "Comet Lake"; case ICL: return "Icelake"; case SKX: diff --git a/cpucounters.h b/cpucounters.h index 8f796eeb..a4e2d9a0 100644 --- a/cpucounters.h +++ b/cpucounters.h @@ -832,7 +832,7 @@ class PCM_API PCM void computeQPISpeedBeckton(int core_nr); void destroyMSR(); void computeNominalFrequency(); - static bool isCPUModelSupported(int model_); + static bool isCPUModelSupported(const int model_); std::string getSupportedUarchCodenames() const; std::string getUnsupportedMessage() const; bool detectModel(); From aed1973e0d6e203a6066b5ce30c952095f645e5f Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Mon, 18 Jan 2021 10:19:26 +0100 Subject: [PATCH 52/60] support Tiger Lake (TGL) addresses https://github.com/opcm/pcm/issues/263 Change-Id: I8837afa632eefe2910c94b05f40f2978ab74a6c0 --- cpucounters.cpp | 3 +++ cpucounters.h | 2 ++ 2 files changed, 5 insertions(+) diff --git a/cpucounters.cpp b/cpucounters.cpp index d376369b..a579241a 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -2079,6 +2079,7 @@ bool PCM::isCPUModelSupported(const int model_) || model_ == KBL_1 || model_ == CML || model_ == ICL + || model_ == TGL || model_ == SKX ); } @@ -3264,6 +3265,8 @@ const char * PCM::getUArchCodename(const int32 cpu_model_param) const return "Comet Lake"; case ICL: return "Icelake"; + case TGL: + return "Tiger Lake"; case SKX: if (cpu_model_param >= 0) { diff --git a/cpucounters.h b/cpucounters.h index a4e2d9a0..86c8a2fd 100644 --- a/cpucounters.h +++ b/cpucounters.h @@ -1255,6 +1255,7 @@ class PCM_API PCM KBL_1 = 142, CML = 166, ICL = 126, + TGL = 140, BDX = 79, KNL = 87, SKL = 94, @@ -1268,6 +1269,7 @@ class PCM_API PCM case PCM::KBL_1: \ case PCM::CML: \ case PCM::ICL: \ + case PCM::TGL: \ case PCM::SKL: private: From 3663129638387bee458200dab59bf6845894b919 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Mon, 18 Jan 2021 11:04:24 +0100 Subject: [PATCH 53/60] fix max IPC for TGL Change-Id: I74d6dacfee695fc15bc660a3fae7aff7dfe4ded5 --- cpucounters.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpucounters.h b/cpucounters.h index 86c8a2fd..b08d774b 100644 --- a/cpucounters.h +++ b/cpucounters.h @@ -1430,7 +1430,7 @@ class PCM_API PCM //! \return max number of instructions per cycle uint32 getMaxIPC() const { - if (ICL == cpu_model) return 5; + if (ICL == cpu_model || TGL == cpu_model) return 5; switch (cpu_model) { case DENVERTON: From be2dbfbc24adfea956fe8d8371e6fd5d31e50a84 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Thu, 28 Jan 2021 12:18:45 +0100 Subject: [PATCH 54/60] support using Linux perf API for core PMU for systems with offlined cores Change-Id: I2f03367d5c2dc0174d946e467a8da6188aaa5c6d --- cpucounters.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpucounters.cpp b/cpucounters.cpp index a579241a..b1cda787 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -2187,11 +2187,13 @@ PCM::ErrorCode PCM::program(const PCM::ProgramMode mode_, const void * parameter canUsePerf = false; std::cerr << "Usage of Linux perf events is disabled through PCM_NO_PERF environment variable. Using direct PMU programming...\n"; } +/* if(num_online_cores < num_cores) { canUsePerf = false; std::cerr << "PCM does not support using Linux perf API on systems with offlined cores. Falling-back to direct PMU programming.\n"; } +*/ else if(PERF_COUNT_HW_MAX <= PCM_PERF_COUNT_HW_REF_CPU_CYCLES) { canUsePerf = false; @@ -2494,6 +2496,7 @@ PCM::ErrorCode PCM::program(const PCM::ProgramMode mode_, const void * parameter // Version for linux/windows/freebsd/dragonflybsd for (int i = 0; i < (int)num_cores; ++i) { + if (isCoreOnline(i) == false) continue; TemporalThreadAffinity tempThreadAffinity(i, false); // speedup trick for Linux const auto status = programCoreCounters(i, mode_, pExtDesc, lastProgrammedCustomCounters[i]); From 91100a80987e696032e9ccc8085c1d7f97a3f5a7 Mon Sep 17 00:00:00 2001 From: Harry van Haaren Date: Thu, 28 Jan 2021 11:59:46 +0000 Subject: [PATCH 55/60] Makefile: add CFLAGS and refactor CXXFLAGS to append to it Adding CFLAGS as the base C options, and appending C++ specific CXXFLAGS values to CFLAGS allows a common set of flags for both C and C++ without duplication. Rework the C example binaries to use $(CFLAGS) when building. Signed-off-by: Harry van Haaren --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index cf4ba704..3f450954 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,8 @@ ifeq ($(UNAME), Linux) EXE += daemon-binaries endif -CXXFLAGS += -Wall -g -O3 -Wno-unknown-pragmas -std=c++11 -fPIC +CFLAGS += -Wall -g -O3 -Wno-unknown-pragmas -fPIC +CXXFLAGS += $(CFLAGS) -std=c++11 # uncomment if your Linux kernel supports access to /dev/mem from user space # CXXFLAGS += -DPCM_USE_PCI_MM_LINUX @@ -83,10 +84,10 @@ libpcm.so: $(COMMON_OBJS) pcm-core.o $(CXX) $(LDFLAGS) $(CXXFLAGS) -DPCM_SILENT -shared $^ $(LIB) -o $@ c_example.x: c_example.c libpcm.so - $(CC) -DPCM_DYNAMIC_LIB $< -ldl -Wl,-rpath,$(shell pwd) -o $@ + $(CC) $(CFLAGS) -DPCM_DYNAMIC_LIB $< -ldl -Wl,-rpath,$(shell pwd) -o $@ c_example_shlib.x: c_example.c libpcm.so - $(CC) $< -L./ -Wl,-rpath,$(shell pwd) -lpcm -o $@ + $(CC) $(CFLAGS) $< -L./ -Wl,-rpath,$(shell pwd) -lpcm -o $@ %.o: %.cpp $(CXX) $(CXXFLAGS) -c $*.cpp -o $*.o From 2ceb1fac8ada36bb627f546a7ea5dba5f736bfe1 Mon Sep 17 00:00:00 2001 From: Harry van Haaren Date: Thu, 28 Jan 2021 11:45:49 +0000 Subject: [PATCH 56/60] c_example: fix rdtscp returning out of bounds CPU This commit fixes an issue where Linux encodes the NUMA ID of a given CPU in bits above 12 of IA32_TSC_AUX[31:0]. The code here was mistakenly including bit 12 and above of the returned value in the CPU integer value. When running the c_example on e.g. NUMA 1 of dual-NUMA platform, bit 12 was set causing an OOB value to be passed to e.g. pcm_c_get_cycles(), resulting in a segfault. Fix issue by removing the NUMA bits from the CPU calculation. Signed-off-by: Harry van Haaren --- c_example.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/c_example.c b/c_example.c index 8b35d1f6..708e34f7 100644 --- a/c_example.c +++ b/c_example.c @@ -11,7 +11,9 @@ int pcm_getcpu() "mov %%ecx, %0\n\t": "=r" (id) :: "%rax", "%rcx", "%rdx"); // processor ID is in ECX: https://www.felixcloutier.com/x86/rdtscp - return id; + // Linux encodes the NUMA node starting at bit 12, so remove the NUMA bits + // when returning the CPU integer by masking with 0xFFF. + return id & 0xFFF; } struct { From 534be46e2fbac174f4b4b5f8fc493eeb295413ce Mon Sep 17 00:00:00 2001 From: Harry van Haaren Date: Thu, 28 Jan 2021 12:14:38 +0000 Subject: [PATCH 57/60] c_example: fix warnings, improve output for beginner user This commit: - fixes an unused-variable on "c" array - Makes whitespace consistent to tabs - Adds [c_example] prints to output to show beginners where what parts of the info logs are coming from. --- c_example.c | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/c_example.c b/c_example.c index 708e34f7..f3dad6e4 100644 --- a/c_example.c +++ b/c_example.c @@ -5,15 +5,15 @@ int pcm_getcpu() { - int id = -1; - asm volatile ( - "rdtscp\n\t" - "mov %%ecx, %0\n\t": - "=r" (id) :: "%rax", "%rcx", "%rdx"); - // processor ID is in ECX: https://www.felixcloutier.com/x86/rdtscp - // Linux encodes the NUMA node starting at bit 12, so remove the NUMA bits - // when returning the CPU integer by masking with 0xFFF. - return id & 0xFFF; + int id = -1; + asm volatile ( + "rdtscp\n\t" + "mov %%ecx, %0\n\t": + "=r" (id) :: "%rax", "%rcx", "%rdx"); + // processor ID is in ECX: https://www.felixcloutier.com/x86/rdtscp + // Linux encodes the NUMA node starting at bit 12, so remove the NUMA + // bits when returning the CPU integer by masking with 0xFFF. + return id & 0xFFF; } struct { @@ -41,13 +41,16 @@ uint64_t pcm_c_get_core_event(uint32_t, uint32_t); int main(int argc, const char *argv[]) { int i,a[100],b[100],c[100]; + uint32_t total = 0; int lcore_id; - for (i=0; i < 100; ++i) - { - a[i] = rand(); - b[i] = rand(); - c[i] = rand(); - } + + /* Seed for predictable rand() results */ + srand(0); + for (i=0; i < 100; ++i) { + a[i] = rand(); + b[i] = rand(); + c[i] = rand(); + } #ifdef PCM_DYNAMIC_LIB void * handle = dlopen("libpcm.so", RTLD_LAZY); @@ -94,12 +97,19 @@ int main(int argc, const char *argv[]) return -2; } + printf("[c_example] Initializing PCM measurements:\n"); PCM.pcm_c_init(); + + printf("[c_example] Calling PCM start()\n"); PCM.pcm_c_start(); for(i=0;i<10000;i++) - c[i%100] = 4 * a[i%100] + b[i%100]; + c[i%100] = 4 * a[i%100] + b[i%100]; + for(i=0;i<100;i++) + total += c[i]; PCM.pcm_c_stop(); + printf("[c_example] PCM measurment stopped, compute result %u\n", total); + lcore_id = pcm_getcpu(); printf("C:%lu I:%lu, IPC:%3.2f\n", PCM.pcm_c_get_cycles(lcore_id), From 981784ba7fd7d6d90ba842c64e9efa4f6d93e618 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 29 Jan 2021 14:53:16 +0100 Subject: [PATCH 58/60] add additional CML model Change-Id: I42568afef265a6db0395358ca1bdf23d4835f05e --- cpucounters.cpp | 1 + cpucounters.h | 1 + 2 files changed, 2 insertions(+) diff --git a/cpucounters.cpp b/cpucounters.cpp index b1cda787..f506d623 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -2090,6 +2090,7 @@ bool PCM::checkModel() if (cpu_model == ATOM_2) cpu_model = ATOM; if (cpu_model == HASWELL_ULT || cpu_model == HASWELL_2) cpu_model = HASWELL; if (cpu_model == BROADWELL_XEON_E3) cpu_model = BROADWELL; + if (cpu_model == CML_1) cpu_model = CML; if(!isCPUModelSupported((int)cpu_model)) { diff --git a/cpucounters.h b/cpucounters.h index b08d774b..ded237e4 100644 --- a/cpucounters.h +++ b/cpucounters.h @@ -1254,6 +1254,7 @@ class PCM_API PCM KBL = 158, KBL_1 = 142, CML = 166, + CML_1 = 165, ICL = 126, TGL = 140, BDX = 79, From e01eec43a397da7c9e736b357d49481abee12afd Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 29 Jan 2021 14:55:03 +0100 Subject: [PATCH 59/60] add additional ICL model Change-Id: I40d17727192e04f6f8f4cfc819fa333d6a3ab581 --- cpucounters.cpp | 1 + cpucounters.h | 1 + 2 files changed, 2 insertions(+) diff --git a/cpucounters.cpp b/cpucounters.cpp index f506d623..afdc4a78 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -2091,6 +2091,7 @@ bool PCM::checkModel() if (cpu_model == HASWELL_ULT || cpu_model == HASWELL_2) cpu_model = HASWELL; if (cpu_model == BROADWELL_XEON_E3) cpu_model = BROADWELL; if (cpu_model == CML_1) cpu_model = CML; + if (cpu_model == ICL_1) cpu_model = ICL; if(!isCPUModelSupported((int)cpu_model)) { diff --git a/cpucounters.h b/cpucounters.h index ded237e4..a86c179d 100644 --- a/cpucounters.h +++ b/cpucounters.h @@ -1256,6 +1256,7 @@ class PCM_API PCM CML = 166, CML_1 = 165, ICL = 126, + ICL_1 = 125, TGL = 140, BDX = 79, KNL = 87, From 290d88713cc270130e2641748cf5e9fc0844235d Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 29 Jan 2021 14:56:49 +0100 Subject: [PATCH 60/60] add additional TGL model Change-Id: If9be91c5a10653dc00c92bb841621d01be8d1b8a --- cpucounters.cpp | 1 + cpucounters.h | 1 + 2 files changed, 2 insertions(+) diff --git a/cpucounters.cpp b/cpucounters.cpp index afdc4a78..e71f95a9 100644 --- a/cpucounters.cpp +++ b/cpucounters.cpp @@ -2092,6 +2092,7 @@ bool PCM::checkModel() if (cpu_model == BROADWELL_XEON_E3) cpu_model = BROADWELL; if (cpu_model == CML_1) cpu_model = CML; if (cpu_model == ICL_1) cpu_model = ICL; + if (cpu_model == TGL_1) cpu_model = TGL; if(!isCPUModelSupported((int)cpu_model)) { diff --git a/cpucounters.h b/cpucounters.h index a86c179d..32f4fadd 100644 --- a/cpucounters.h +++ b/cpucounters.h @@ -1258,6 +1258,7 @@ class PCM_API PCM ICL = 126, ICL_1 = 125, TGL = 140, + TGL_1 = 141, BDX = 79, KNL = 87, SKL = 94,