OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
DynamicWatchdog.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <atomic>
18 #include <chrono>
19 #include <thread>
20 
21 #include "DynamicWatchdog.h"
22 #include "Logger/Logger.h"
23 #include "Shared/funcannotations.h"
24 
25 static FORCE_INLINE uint64_t read_cycle_counter(void) {
26 #if (defined(__x86_64__) || defined(__x86_64))
27  unsigned hi, lo;
28  __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi));
29  return (static_cast<uint64_t>(hi) << 32) | static_cast<uint64_t>(lo);
30 #else
31  // Plug in other architectures' cycle counter reads, e.g. MRC on ARM
32  return 0LL;
33 #endif
34 }
35 
36 extern "C" RUNTIME_EXPORT uint64_t dynamic_watchdog_init(unsigned ms_budget) {
37  static uint64_t dw_cycle_start = 0ULL;
38  static uint64_t dw_cycle_budget = 0ULL;
39  static std::atomic_bool dw_abort{false};
40 
41  if (ms_budget == static_cast<unsigned>(DW_DEADLINE)) {
42  if (dw_abort.load()) {
43  { return 0LL; }
44  }
45  return dw_cycle_start + dw_cycle_budget;
46  }
47  if (ms_budget == static_cast<unsigned>(DW_ABORT)) {
48  dw_abort = true;
49  return 0LL;
50  }
51  if (ms_budget == static_cast<unsigned>(DW_RESET)) {
52  dw_abort = false;
53  return 0LL;
54  }
55 
56  // Init cycle start, measure freq, set and return cycle budget
57  dw_cycle_start = read_cycle_counter();
58  std::this_thread::sleep_for(std::chrono::milliseconds(1));
59  auto freq_kHz = read_cycle_counter() - dw_cycle_start;
60  dw_cycle_budget = freq_kHz * static_cast<uint64_t>(ms_budget);
61  VLOG(1) << "INIT: thread " << std::this_thread::get_id() << ": ms_budget " << ms_budget
62  << ", cycle_start " << dw_cycle_start << ", cycle_budget " << dw_cycle_budget
63  << ", dw_deadline " << dw_cycle_start + dw_cycle_budget;
64  return dw_cycle_budget;
65 }
66 
67 // timeout detection
68 extern "C" RUNTIME_EXPORT bool dynamic_watchdog() {
69  auto clock = read_cycle_counter();
70  auto dw_deadline = dynamic_watchdog_init(static_cast<unsigned>(DW_DEADLINE));
71  if (clock > dw_deadline) {
72  LOG(INFO) << "TIMEOUT: thread " << std::this_thread::get_id() << ": clock " << clock
73  << ", deadline " << dw_deadline;
74  return true;
75  }
76  return false;
77 }
__device__ bool dynamic_watchdog()
#define LOG(tag)
Definition: Logger.h:285
__device__ int64_t dw_cycle_budget
Definition: cuda_mapd_rt.cu:93
RUNTIME_EXPORT uint64_t dynamic_watchdog_init(unsigned ms_budget)
#define RUNTIME_EXPORT
static FORCE_INLINE uint64_t read_cycle_counter(void)
#define FORCE_INLINE
__device__ int32_t dw_abort
Definition: cuda_mapd_rt.cu:94
#define VLOG(n)
Definition: Logger.h:388