OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
DynamicWatchdog.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <atomic>
18 #include <chrono>
19 #include <thread>
20 
21 #include "DynamicWatchdog.h"
22 #include "Logger/Logger.h"
23 #include "Shared/funcannotations.h"
24 
25 static FORCE_INLINE uint64_t read_cycle_counter(void) {
26 #if (defined(__x86_64__) || defined(__x86_64))
27  unsigned hi, lo;
28  __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi));
29  return (static_cast<uint64_t>(hi) << 32) | static_cast<uint64_t>(lo);
30 #else
31  // Plug in other architectures' cycle counter reads, e.g. MRC on ARM
32  return 0LL;
33 #endif
34 }
35 
36 extern "C" RUNTIME_EXPORT uint64_t dynamic_watchdog_init(unsigned ms_budget) {
37  static uint64_t dw_cycle_start = 0ULL;
38  static uint64_t dw_cycle_budget = 0ULL;
39  static std::atomic_bool dw_abort{false};
40 
41  if (ms_budget == static_cast<unsigned>(DW_DEADLINE)) {
42  if (dw_abort.load()) {
43  {
44  return 0LL;
45  }
46  }
47  return dw_cycle_start + dw_cycle_budget;
48  }
49  if (ms_budget == static_cast<unsigned>(DW_ABORT)) {
50  dw_abort = true;
51  return 0LL;
52  }
53  if (ms_budget == static_cast<unsigned>(DW_RESET)) {
54  dw_abort = false;
55  return 0LL;
56  }
57 
58  // Init cycle start, measure freq, set and return cycle budget
59  dw_cycle_start = read_cycle_counter();
60  std::this_thread::sleep_for(std::chrono::milliseconds(1));
61  auto freq_kHz = read_cycle_counter() - dw_cycle_start;
62  dw_cycle_budget = freq_kHz * static_cast<uint64_t>(ms_budget);
63  VLOG(1) << "INIT: thread " << std::this_thread::get_id() << ": ms_budget " << ms_budget
64  << ", cycle_start " << dw_cycle_start << ", cycle_budget " << dw_cycle_budget
65  << ", dw_deadline " << dw_cycle_start + dw_cycle_budget;
66  return dw_cycle_budget;
67 }
68 
69 // timeout detection
70 extern "C" RUNTIME_EXPORT bool dynamic_watchdog() {
71  auto clock = read_cycle_counter();
72  auto dw_deadline = dynamic_watchdog_init(static_cast<unsigned>(DW_DEADLINE));
73  if (clock > dw_deadline) {
74  LOG(INFO) << "TIMEOUT: thread " << std::this_thread::get_id() << ": clock " << clock
75  << ", deadline " << dw_deadline;
76  return true;
77  }
78  return false;
79 }
__device__ bool dynamic_watchdog()
#define LOG(tag)
Definition: Logger.h:285
__device__ int64_t dw_cycle_budget
Definition: cuda_mapd_rt.cu:93
RUNTIME_EXPORT uint64_t dynamic_watchdog_init(unsigned ms_budget)
#define RUNTIME_EXPORT
static FORCE_INLINE uint64_t read_cycle_counter(void)
#define FORCE_INLINE
__device__ int32_t dw_abort
Definition: cuda_mapd_rt.cu:94
#define VLOG(n)
Definition: Logger.h:387