Add timer object for timestamps

.clang-tidy

@@ -26,6 +26,7 @@ Checks: >
   -hicpp-no-array-decay,
   -hicpp-signed-bitwise,
   -llvm-header-guard,
+  -llvm-include-order,
   -llvmlibc-*,
   -misc-non-private-member-variables-in-classes,
   -misc-unused-parameters,
@@ -63,6 +64,9 @@ Checks: >
 # llvm-header-guard
 #   Doesn't seem to be satisfied easily
 #
+# llvm-include-order
+#   clang-format takes care of the include order
+#
 # llvmlibc-*
 #   These guidelines are particular to llvm-libc and unnecessary here
 #

src/amdinfer/batching/CMakeLists.txt

@@ -31,6 +31,8 @@ foreach(file ${files})
   list(APPEND targets ${target})
 endforeach()
 
+target_link_libraries(softBatcher INTERFACE timer)
+
 add_library(batching STATIC ${BATCHING_OBJS})
 target_link_libraries(batching INTERFACE batcher ${targets})
 

src/amdinfer/batching/soft.cpp

@@ -21,7 +21,6 @@
 #include "amdinfer/batching/soft.hpp"
 
 #include <algorithm>  // for max
-#include <chrono>     // for milliseconds, duration_cast
 #include <cstddef>    // for size_t
 #include <cstdint>    // for int32_t
 #include <memory>     // for unique_ptr, allocator
@@ -40,11 +39,10 @@
 #include "amdinfer/observation/tracing.hpp"  // for Trace
 #include "amdinfer/util/queue.hpp"           // for BlockingConcurrentQueue
 #include "amdinfer/util/thread.hpp"          // for setThreadName
+#include "amdinfer/util/timer.hpp"           // for Timer
 
-using std::chrono::duration_cast;
-using std::chrono::milliseconds;
-
-constexpr auto kDefaultTimeout = milliseconds(100);
+// default batcher timeout in milliseconds
+constexpr auto kDefaultTimeout = 100;
 
 namespace amdinfer {
 
@@ -57,10 +55,9 @@ void SoftBatcher::doRun(WorkerInfo* worker) {
 
   bool run = true;
 
-  auto timeout = duration_cast<milliseconds>(kDefaultTimeout);
+  auto timeout = kDefaultTimeout;
   if (this->parameters_.has("timeout")) {
-    timeout = duration_cast<milliseconds>(
-      milliseconds(this->parameters_.get<int32_t>("timeout")));
+    timeout = this->parameters_.get<int32_t>("timeout");
   }
 
   while (run) {
@@ -81,20 +78,21 @@ void SoftBatcher::doRun(WorkerInfo* worker) {
 #endif
 
     bool first_request = true;
-    auto start_time = std::chrono::high_resolution_clock::now();
+    util::Timer timer{true};
 
     do {
       InterfacePtr req;
       if (first_request) {
         // wait for the first request
         this->input_queue_->wait_dequeue(req);
-        start_time = std::chrono::high_resolution_clock::now();
+        timer.add("start");
         AMDINFER_LOG_DEBUG(logger,
                            "Got request of a new batch for " + this->model_);
       } else {
-        auto remaining_time =
-          timeout - (std::chrono::high_resolution_clock::now() - start_time);
-        auto duration = std::max(remaining_time, std::chrono::nanoseconds(0));
+        timer.stop();
+
+        auto remaining_time = timeout - timer.count<std::milli, int>();
+        auto duration = std::max(remaining_time, 0);
         bool valid = this->input_queue_->wait_dequeue_timed(req, duration);
         if (!valid) {
           break;

src/amdinfer/observation/CMakeLists.txt

@@ -29,7 +29,7 @@ endif()
 
 if(${AMDINFER_ENABLE_METRICS})
   add_library(metrics OBJECT metrics.cpp)
-  target_link_libraries(metrics INTERFACE prometheus-cpp-core)
+  target_link_libraries(metrics INTERFACE prometheus-cpp-core util)
   target_include_directories(metrics PRIVATE ${AMDINFER_INCLUDE_DIRS})
   enable_ipo_on_target(metrics)
 

src/amdinfer/observation/metrics.cpp

@@ -26,15 +26,16 @@
 #include <prometheus/gauge.h>            // for Gauge, BuildGauge
 #include <prometheus/metric_family.h>    // for MetricFamily
 #include <prometheus/serializer.h>       // for Serializer
+#include <prometheus/summary.h>          // for CKMSQuantiles, CKMSQuantiles...
 #include <prometheus/text_serializer.h>  // for TextSerializer
 
-#include <chrono>    // for microseconds, duration, dura...
 #include <iterator>  // for move_iterator, make_move_ite...
 #include <memory>    // for weak_ptr, allocator, shared_ptr
+#include <ratio>     // for micro
 #include <string>    // for string
 #include <vector>    // for vector
 
-#include "prometheus/summary.h"  // for CKMSQuantiles, CKMSQuantiles...
+#include "amdinfer/util/timer.hpp"  // for Timer
 
 namespace amdinfer {
 
@@ -217,7 +218,7 @@ void Metrics::observeSummary(MetricSummaryIDs id, double value) {
 }
 
 std::string Metrics::getMetrics() {
-  auto start_time_of_request = std::chrono::steady_clock::now();
+  util::Timer timer{true};
 
   std::vector<prometheus::MetricFamily> metrics;
 
@@ -239,10 +240,9 @@ std::string Metrics::getMetrics() {
   std::string response = serializer_->Serialize(metrics);
   auto body_size = response.length();
 
-  auto stop_time_of_request = std::chrono::steady_clock::now();
-  auto duration = std::chrono::duration_cast<std::chrono::microseconds>(
-    stop_time_of_request - start_time_of_request);
-  this->observeSummary(MetricSummaryIDs::MetricLatency, duration.count());
+  timer.stop();
+  auto duration = timer.count<std::micro>();
+  this->observeSummary(MetricSummaryIDs::MetricLatency, duration);
 
   this->bytes_transferred_.increment(MetricCounterIDs::TransferredBytes,
                                      body_size);

src/amdinfer/util/CMakeLists.txt

@@ -21,6 +21,7 @@ list(
          parse_env
          exec
          read_nth_line
+         timer
 )
 
 list(APPEND UTIL_OBJS)

src/amdinfer/util/timer.cpp

@@ -0,0 +1,53 @@
+// Copyright 2022 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file
+ * @brief Implements a helper timer class
+ */
+
+#include "amdinfer/util/timer.hpp"
+
+#include <utility>  // for pair
+
+namespace amdinfer::util {
+
+TimePoint getTime() { return std::chrono::high_resolution_clock::now(); }
+
+Timer::Timer(TimePoint time) { add("start", time); }
+
+Timer::Timer(bool start) {
+  if (start) {
+    this->start();
+  }
+}
+
+void Timer::start() { add("start"); }
+
+void Timer::stop() { add("stop"); }
+
+void Timer::add(const std::string& label) { add(label, getTime()); }
+
+void Timer::add(const std::string& label, TimePoint time) {
+  auto point = times_.find(label);
+  if (point == times_.end()) {
+    times_.try_emplace(label, time);
+  } else {
+    point->second = time;
+  }
+}
+
+void Timer::clear() { times_.clear(); }
+
+}  // namespace amdinfer::util

src/amdinfer/util/timer.hpp

@@ -0,0 +1,105 @@
+// Copyright 2022 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @file
+ * @brief Defines a helper timer class
+ */
+
+#ifndef GUARD_AMDINFER_UTIL_TIMER
+#define GUARD_AMDINFER_UTIL_TIMER
+
+#include <chrono>         // IWYU pragma: export
+#include <ratio>          // for ratio
+#include <string>         // for string
+#include <unordered_map>  // for unordered_map
+
+namespace amdinfer::util {
+
+using TimePoint = std::chrono::_V2::system_clock::time_point;
+
+TimePoint getTime();
+
+class Timer {
+ public:
+  /**
+   * @brief Construct a new Timer object
+   *
+   * @param time set the start time to this value
+   */
+  explicit Timer(TimePoint time);
+  /**
+   * @brief Construct a new Timer object
+   *
+   * @param start start the timer on construction
+   */
+  explicit Timer(bool start = false);
+
+  /**
+   * @brief Add or replace the label "start" with the current time
+   *
+   */
+  void start();
+  /**
+   * @brief Add or replace the label "stop" with the current time
+   *
+   */
+  void stop();
+  /**
+   * @brief Add or replace the given label with the current time
+   *
+   * @param label label to assign to the current timestamp
+   */
+  void add(const std::string& label);
+  /**
+   * @brief Add or replace the given label with the given time
+   *
+   * @param label label to assign to the timestamp
+   * @param time the timestamp
+   */
+  void add(const std::string& label, TimePoint time);
+
+  /**
+   * @brief Clear all the saved timestamps
+   *
+   */
+  void clear();
+
+  /**
+   * @brief Return the duration between two labeled timestamps
+   *
+   * @tparam U the ratio to convert the time e.g. std::micro for microseconds
+   * @tparam T the type to return the time as
+   * @param start the label for the start time
+   * @param stop the label for the stop time
+   * @return T the duration between the start and stop time in the right units
+   */
+  template <typename U = std::ratio<1, 1>, typename T = double>
+  T count(const std::string& start = "start",
+          const std::string& stop = "stop") {
+    const auto& start_time = times_.at(start);
+    const auto& stop_time = times_.at(stop);
+
+    auto duration = std::chrono::duration_cast<std::chrono::duration<T, U>>(
+      stop_time - start_time);
+    return duration.count();
+  }
+
+ private:
+  std::unordered_map<std::string, TimePoint> times_;
+};
+
+}  // namespace amdinfer::util
+
+#endif  // GUARD_AMDINFER_UTIL_TIMER

src/amdinfer/workers/CMakeLists.txt

@@ -58,7 +58,7 @@ foreach(worker ${workers})
   add_library(${worker_name} SHARED ${file_name}.cpp)
   target_link_options(${worker_name} PUBLIC "LINKER:-E")
   target_link_libraries(
-    ${worker_name} PRIVATE batching buffers core observation
+    ${worker_name} PRIVATE batching buffers core observation timer
   )
   target_include_directories(${worker_name} PRIVATE ${AMDINFER_INCLUDE_DIRS})
   enable_ipo_on_target(${worker_name})
@@ -129,6 +129,7 @@ if(${AMDINFER_ENABLE_AKS})
               core
               base64
               parse_env
+              timer
               observation
               aks
               vart-runner
@@ -153,7 +154,7 @@ if(${AMDINFER_ENABLE_AKS})
   target_link_options(workerAks PUBLIC "LINKER:-E")
   target_link_libraries(
     workerAks PRIVATE batching buffers core parse_env observation
-    PRIVATE aks vart-runner xir
+    PRIVATE aks vart-runner xir timer
   )
   target_include_directories(workerAks PRIVATE ${AMDINFER_INCLUDE_DIRS})
   enable_ipo_on_target(workerAks)

src/amdinfer/workers/aks.cpp

@@ -22,12 +22,12 @@
 #include <aks/AksTensorBuffer.h>   // for AksTensorBuffer
 
 #include <algorithm>               // for max
-#include <chrono>                  // for microseconds, duration_...
 #include <cstddef>                 // for size_t, byte
 #include <cstdint>                 // for int32_t
 #include <cstring>                 // for memcpy
 #include <future>                  // for future
 #include <memory>                  // for unique_ptr, allocator
+#include <ratio>                   // for micro
 #include <string>                  // for string, operator+, char...
 #include <thread>                  // for thread
 #include <utility>                 // for move, pair
@@ -48,6 +48,7 @@
 #include "amdinfer/observation/tracing.hpp"    // for Trace
 #include "amdinfer/util/parse_env.hpp"         // for autoExpandEnvironmentVa...
 #include "amdinfer/util/thread.hpp"            // for setThreadName
+#include "amdinfer/util/timer.hpp"             // for Timer
 #include "amdinfer/workers/worker.hpp"         // for Worker, kNumBufferAuto
 
 namespace AKS {  // NOLINT(readability-identifier-naming)
@@ -193,11 +194,11 @@ void Aks::doRun(BatchPtrQueue* input_queue) {
       }
 
 #ifdef AMDINFER_ENABLE_METRICS
-      auto duration = std::chrono::duration_cast<std::chrono::microseconds>(
-        std::chrono::high_resolution_clock::now() -
-        batch->getTime(static_cast<int>(j)));
+      util::Timer timer{batch->getTime(j)};
+      timer.stop();
+      auto duration = timer.count<std::micro>();
       Metrics::getInstance().observeSummary(MetricSummaryIDs::RequestLatency,
-                                            duration.count());
+                                            duration);
 #endif
 #ifdef AMDINFER_ENABLE_TRACING
       auto context = trace->propagate();