logger Namespace Reference

Namespaces
	anonymous_namespace{Logger.cpp}

Classes
class	Duration
class	DurationTree
struct	GetDepth
class	JsonEncoder
struct	EraseDurationTrees
	Depth-first search and erase all DurationTrees. Not thread-safe. More...
class	LogOptions
struct	LogShutdown
class	Logger
class	DebugTimer
class	QidScopeGuard

Typedefs
using	ChannelLogger = boost::log::sources::channel_logger_mt< Channel >
using	SeverityLogger = boost::log::sources::severity_logger_mt< Severity >
using	ClogSync = sinks::synchronous_sink< sinks::text_ostream_backend >
using	FileSync = sinks::synchronous_sink< sinks::text_file_backend >
using	Clock = std::chrono::steady_clock
using	DurationTreeNode = boost::variant< Duration, DurationTree & >
using	DurationTreeNodes = std::deque< DurationTreeNode >
using	DurationTreeMap = std::unordered_map< ThreadId, std::unique_ptr< DurationTree >>
using	Channels = std::set< Channel >
using	FatalFunc = void(*)() noexcept
using	QueryId = uint64_t
using	ThreadId = uint64_t

Enumerations
enum	Channel { IR = 0, PTX, ASM, _NCHANNELS }
enum	Severity {   DEBUG4 = 0, DEBUG3, DEBUG2, DEBUG1,   INFO, WARNING, ERROR, FATAL,   _NSEVERITIES }

Functions
	BOOST_LOG_GLOBAL_LOGGER_CTOR_ARGS (gChannelLogger_IR, ChannelLogger,(keywords::channel=IR)) BOOST_LOG_GLOBAL_LOGGER_CTOR_ARGS(gChannelLogger_PTX
template<typename CONSOLE_SINK >
boost::shared_ptr< CONSOLE_SINK >	make_sink (LogOptions const &log_opts)
void	init (LogOptions const &log_opts)
void	set_once_fatal_func (FatalFunc fatal_func)
void	shutdown ()
std::istream &	operator>> (std::istream &in, Channels &channels)
std::ostream &	operator<< (std::ostream &out, Channels const &channels)
std::istream &	operator>> (std::istream &in, Severity &sev)
std::ostream &	operator<< (std::ostream &out, Severity const &sev)
QueryId	query_id ()
QidScopeGuard	set_thread_local_query_id (QueryId const query_id)
template<typename... Ts>
Duration *	newDuration (Severity severity, Ts &&...args)
std::ostream &	operator<< (std::ostream &os, Duration const &duration)
std::ostream &	operator<< (std::ostream &os, DurationTree const &duration_tree)
boost::log::record_ostream &	operator<< (boost::log::record_ostream &os, DurationTreeMap::const_reference kv_pair)
void	logAndEraseDurationTree (std::string *json_str)
void	debug_timer_new_thread (ThreadId parent_thread_id)
ThreadId	thread_id ()
boost::filesystem::path	get_log_dir_path ()
bool	fast_logging_check (Channel)
bool	fast_logging_check (Severity severity)
template<typename X , typename Y >
BOOST_NOINLINE std::string *	check_failed (X const &x, Y const &y, char const *xstr, char const *ystr, char const *op_str)

Variables
std::once_flag	g_fatal_func_flag
bool	g_any_active_channels {false}
Severity	g_min_active_severity {Severity::FATAL}
static boost::filesystem::path	g_log_dir_path
thread_local std::atomic< QueryId >	g_query_id {0}
std::mutex	g_duration_tree_map_mutex
DurationTreeMap	g_duration_tree_map
std::atomic< ThreadId >	g_next_thread_id {0}
thread_local ThreadId	g_thread_id = g_next_thread_id++
constexpr std::array< char const *, 3 >	ChannelNames {"IR", "PTX", "ASM"}
constexpr std::array< char, 3 >	ChannelSymbols {'R', 'P', 'A'}
constexpr std::array< char const *, 8 >	SeverityNames
constexpr std::array< char, 8 >	SeveritySymbols {'4', '3', '2', '1', 'I', 'W', 'E', 'F'}

Typedef Documentation

logger::ChannelLogger

Definition at line 53 of file Logger.cpp.

using logger::Channels = typedef std::set<Channel>

Definition at line 130 of file Logger.h.

using logger::Clock = typedef std::chrono::steady_clock

Definition at line 499 of file Logger.cpp.

using logger::ClogSync = typedef sinks::synchronous_sink<sinks::text_ostream_backend>

Definition at line 288 of file Logger.cpp.

using logger::DurationTreeMap = typedef std::unordered_map<ThreadId, std::unique_ptr<DurationTree>>

Definition at line 594 of file Logger.cpp.

using logger::DurationTreeNode = typedef boost::variant<Duration, DurationTree&>

Definition at line 537 of file Logger.cpp.

using logger::DurationTreeNodes = typedef std::deque<DurationTreeNode>

Definition at line 538 of file Logger.cpp.

using logger::FatalFunc = typedef void (*)() noexcept

Definition at line 175 of file Logger.h.

using logger::FileSync = typedef sinks::synchronous_sink<sinks::text_file_backend>

Definition at line 289 of file Logger.cpp.

using logger::QueryId = typedef uint64_t

Definition at line 335 of file Logger.h.

typedef boost::log::sources::severity_logger_mt< Severity > logger::SeverityLogger

Definition at line 58 of file Logger.cpp.

using logger::ThreadId = typedef uint64_t

Definition at line 364 of file Logger.h.

Enumeration Type Documentation

enum logger::Channel

Enumerator
IR
PTX
ASM
_NCHANNELS

Definition at line 88 of file Logger.h.

{ IR = 0, PTX, ASM, _NCHANNELS };

enum logger::Severity

Enumerator
DEBUG4
DEBUG3
DEBUG2
DEBUG1
INFO
WARNING
ERROR
FATAL
_NSEVERITIES

Definition at line 100 of file Logger.h.

              {
  DEBUG4 = 0,
  DEBUG3,
  DEBUG2,
  DEBUG1,
  INFO,
  WARNING,
  ERROR,
  FATAL,
  _NSEVERITIES  // number of severity levels
};

Function Documentation

logger::BOOST_LOG_GLOBAL_LOGGER_CTOR_ARGS	(	gChannelLogger_IR	,
		ChannelLogger	,
		(keywords::channel=IR)
	)

template<typename X , typename Y >

BOOST_NOINLINE std::string* logger::check_failed	(	X const &	x,
		Y const &	y,
		char const *	xstr,
		char const *	ystr,
		char const *	op_str
	)

Definition at line 238 of file Logger.h.

                                                             {
  std::stringstream ss;
  ss << "Check failed: " << xstr << op_str << ystr << " (" << x << op_str << y << ") ";
  return new std::string(ss.str());  // Deleted by CHECK_OP macro.
}

void logger::debug_timer_new_thread

(

ThreadId

parent_thread_id

)

Call this when a new thread is spawned that will have timers that need to be associated with timers on the parent thread.

Definition at line 795 of file Logger.cpp.

References CHECK, g_duration_tree_map, g_duration_tree_map_mutex, g_next_thread_id, and g_thread_id.

                                                       {
  std::lock_guard<std::mutex> lock_guard(g_duration_tree_map_mutex);
  auto parent_itr = g_duration_tree_map.find(parent_thread_id);
  CHECK(parent_itr != g_duration_tree_map.end()) << parent_thread_id;
  auto const current_depth = parent_itr->second->currentDepth();
  auto& duration_tree_ptr = g_duration_tree_map[g_thread_id];
  if (!duration_tree_ptr) {
    duration_tree_ptr = std::make_unique<DurationTree>(g_thread_id, current_depth + 1);
    parent_itr->second->pushDurationTree(*duration_tree_ptr);
  } else if (parent_thread_id == g_thread_id) {
    // DEBUG_TIMER_NEW_THREAD() was called but this is actually the same thread as the
    // parent, so nothing will be done. TBB does this.
  } else {
    // This appears to be a recycled thread, so g_thread_id is updated manually.
    // It is assumed the prior thread that was using this g_thread_id has completed,
    // but we need to use a different value since the old value still exists as a
    // key in g_duration_tree_map.
    g_thread_id = g_next_thread_id++;
    auto const insert = g_duration_tree_map.insert(
        {g_thread_id, std::make_unique<DurationTree>(g_thread_id, current_depth + 1)});
    CHECK(insert.second) << parent_thread_id << " -> " << g_thread_id;
    parent_itr->second->pushDurationTree(*insert.first->second);
  }
}

bool logger::fast_logging_check ( Channel  )

inline

Definition at line 196 of file Logger.h.

References g_any_active_channels.

Referenced by anonymous_namespace{RelAlgOptimizer.cpp}::cleanup_dead_nodes(), fold_filters(), query_state::StdLog::log(), and query_state::StdLog::logCallStack().

                                        {
  extern bool g_any_active_channels;
  return g_any_active_channels;
}

Here is the caller graph for this function:

bool logger::fast_logging_check ( Severity  severity )

inline

Definition at line 201 of file Logger.h.

References g_min_active_severity.

                                                  {
  extern Severity g_min_active_severity;
  return g_min_active_severity <= severity;
}

boost::filesystem::path logger::get_log_dir_path

(

)

Definition at line 824 of file Logger.cpp.

References g_log_dir_path.

Referenced by Catalog_Namespace::Catalog::initializeSystemServers().

                                       {
  return boost::filesystem::canonical(g_log_dir_path);
}

Here is the caller graph for this function:

void logger::init

(

LogOptions const &

log_opts

)

Definition at line 308 of file Logger.cpp.

References _NSEVERITIES, logger::LogOptions::channels_, logger::LogOptions::full_log_dir(), g_any_active_channels, g_log_dir_path, g_min_active_severity, INFO, nvtx_helpers::init(), LOG, logger::LogOptions::max_files_, logger::LogOptions::severity_, and logger::LogOptions::severity_clog_.

Referenced by UdfCompiler::compileFromCommandLine(), UdfCompiler::generateAST(), QueryRunner::QueryRunner::init(), CommandLineOptions::init_logging(), main(), ColumnarResults::mergeResults(), ResultSetManager::reduce(), Executor::reduceMultiDeviceResultSets(), FixedLengthEncoder< T, V >::updateStatsEncoded(), and NoneEncoder< T >::updateStatsEncoded().

                                      {
  boost::shared_ptr<boost::log::core> core = boost::log::core::get();
  // boost::log::add_common_attributes(); // LineID TimeStamp ProcessID ThreadID
  core->add_global_attribute("TimeStamp", attr::local_clock());
  core->add_global_attribute("ProcessID", attr::current_process_id());
  if (0 < log_opts.max_files_) {
    boost::filesystem::path const full_log_dir = log_opts.full_log_dir();
    bool const log_dir_was_created = boost::filesystem::create_directory(full_log_dir);
    // Don't create separate log sinks for anything less than Severity::INFO.
    Severity const min_sink_level = std::max(Severity::INFO, log_opts.severity_);
    for (int i = min_sink_level; i < Severity::_NSEVERITIES; ++i) {
      Severity const level = static_cast<Severity>(i);
      core->add_sink(make_sink<FileSync>(log_opts, full_log_dir, level));
    }
    g_min_active_severity = std::min(g_min_active_severity, log_opts.severity_);
    if (log_dir_was_created) {
      LOG(INFO) << "Log directory(" << full_log_dir.native() << ") created.";
    }
    for (auto const channel : log_opts.channels_) {
      core->add_sink(make_sink<FileSync>(log_opts, full_log_dir, channel));
    }
    g_any_active_channels = !log_opts.channels_.empty();
  }
  core->add_sink(make_sink<ClogSync>(log_opts));
  g_min_active_severity = std::min(g_min_active_severity, log_opts.severity_clog_);
  nvtx_helpers::init();
  g_log_dir_path = log_opts.full_log_dir();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void logger::logAndEraseDurationTree

(

std::string *

json_str

)

Definition at line 747 of file Logger.cpp.

References CHECK, g_duration_tree_map, g_duration_tree_map_mutex, g_thread_id, and logger::JsonEncoder::str().

Referenced by logger::DebugTimer::stop(), and logger::DebugTimer::stopAndGetJson().

                                                  {
  std::lock_guard<std::mutex> lock_guard(g_duration_tree_map_mutex);
  DurationTreeMap::const_iterator const itr = g_duration_tree_map.find(g_thread_id);
  CHECK(itr != g_duration_tree_map.cend());
  auto const& root_duration = itr->second->rootDuration();
  if (auto log = Logger(root_duration.severity_)) {
    log.stream(root_duration.file_, root_duration.line_) << *itr;
  }
  if (json_str) {
    JsonEncoder json_encoder;
    *json_str = json_encoder.str(*itr);
  }
  EraseDurationTrees erase_duration_trees;
  erase_duration_trees(itr);
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename CONSOLE_SINK >

boost::shared_ptr<CONSOLE_SINK> logger::make_sink

(

LogOptions const &

log_opts

)

Definition at line 292 of file Logger.cpp.

References logger::LogOptions::severity_clog_.

                                                                    {
  auto sink = boost::make_shared<CONSOLE_SINK>();
  boost::shared_ptr<std::ostream> clog(&std::clog, boost::null_deleter());
  sink->locked_backend()->add_stream(clog);
  sink->set_filter(log_opts.severity_clog_ <= severity);
  set_formatter<Severity>(sink);
  return sink;
}

template<typename... Ts>

Duration* logger::newDuration	(	Severity	severity,
		Ts &&...	args
	)

Definition at line 602 of file Logger.cpp.

References run_benchmark_import::args, g_duration_tree_map, g_duration_tree_map_mutex, g_enable_debug_timer, and g_thread_id.

                                                       {
  if (g_enable_debug_timer) {
    std::lock_guard<std::mutex> lock_guard(g_duration_tree_map_mutex);
    auto& duration_tree_ptr = g_duration_tree_map[g_thread_id];
    if (!duration_tree_ptr) {
      duration_tree_ptr = std::make_unique<DurationTree>(g_thread_id, 0);
    }
    return duration_tree_ptr->newDuration(severity, std::forward<Ts>(args)...);
  }
  return nullptr;  // Inactive - don't measure or report timing.
}

std::ostream& logger::operator<<	(	std::ostream &	out,
		Channels const &	channels
	)

Definition at line 387 of file Logger.cpp.

References ChannelNames.

                                                                  {
  int i = 0;
  for (auto const channel : channels) {
    out << (i++ ? " " : "") << ChannelNames.at(channel);
  }
  return out;
}

std::ostream& logger::operator<<	(	std::ostream &	out,
		Severity const &	sev
	)

Definition at line 410 of file Logger.cpp.

                                                             {
  return out << SeverityNames.at(sev);
}

std::ostream& logger::operator<<	(	std::ostream &	os,
		Duration const &	duration
	)

Definition at line 614 of file Logger.cpp.

References logger::Duration::depth_, logger::Duration::file_, nvtx_helpers::anonymous_namespace{nvtx_helpers.cpp}::filename(), logger::Duration::line_, logger::Duration::name_, logger::Duration::relative_start_time(), and logger::Duration::value().

                                                                 {
  return os << std::setw(2 * duration.depth_) << ' ' << duration.value() << "ms start("
            << duration.relative_start_time() << "ms) " << duration.name_ << ' '
            << filename(duration.file_) << ':' << duration.line_;
}

Here is the call graph for this function:

std::ostream& logger::operator<<	(	std::ostream &	os,
		DurationTree const &	duration_tree
	)

Definition at line 620 of file Logger.cpp.

References logger::DurationTree::depth_, logger::DurationTree::durations(), and logger::DurationTree::thread_id_.

                                                                          {
  os << std::setw(2 * duration_tree.depth_) << ' ' << "New thread("
     << duration_tree.thread_id_ << ')';
  for (auto const& duration_tree_node : duration_tree.durations()) {
    os << '\n' << duration_tree_node;
  }
  return os << '\n'
            << std::setw(2 * duration_tree.depth_) << ' ' << "End thread("
            << duration_tree.thread_id_ << ')';
}

Here is the call graph for this function:

boost::log::record_ostream& logger::operator<<	(	boost::log::record_ostream &	os,
		DurationTreeMap::const_reference	kv_pair
	)

Definition at line 632 of file Logger.cpp.

                                                                               {
  auto itr = kv_pair.second->durations().cbegin();
  auto const end = kv_pair.second->durations().cend();
  auto const& root_duration = boost::get<Duration>(*itr);
  os << "DEBUG_TIMER thread_id(" << kv_pair.first << ")\n"
     << root_duration.value() << "ms total duration for " << root_duration.name_;
  for (++itr; itr != end; ++itr) {
    os << '\n' << *itr;
  }
  return os;
}

std::istream& logger::operator>>	(	std::istream &	in,
		Channels &	channels
	)

Definition at line 367 of file Logger.cpp.

References ChannelNames, parse_ast::line, and logger::anonymous_namespace{Logger.cpp}::unquote().

                                                           {
  std::string line;
  std::getline(in, line);
  unquote(line);
  std::regex const rex(R"(\w+)");
  using TokenItr = std::regex_token_iterator<std::string::iterator>;
  TokenItr const end;
  for (TokenItr tok(line.begin(), line.end(), rex); tok != end; ++tok) {
    auto itr = std::find(ChannelNames.cbegin(), ChannelNames.cend(), *tok);
    if (itr == ChannelNames.cend()) {
      in.setstate(std::ios_base::failbit);
      break;
    } else {
      channels.emplace(static_cast<Channel>(itr - ChannelNames.cbegin()));
    }
  }
  return in;
}

Here is the call graph for this function:

std::istream& logger::operator>>	(	std::istream &	in,
		Severity &	sev
	)

Definition at line 396 of file Logger.cpp.

References logger::anonymous_namespace{Logger.cpp}::unquote().

                                                      {
  std::string token;
  in >> token;
  unquote(token);
  auto itr = std::find(SeverityNames.cbegin(), SeverityNames.cend(), token);
  if (itr == SeverityNames.cend()) {
    in.setstate(std::ios_base::failbit);
  } else {
    sev = static_cast<Severity>(itr - SeverityNames.cbegin());
  }
  return in;
}

Here is the call graph for this function:

QueryId logger::query_id

(

)

Definition at line 473 of file Logger.cpp.

References g_query_id.

Referenced by DBHandler::broadcast_serialized_rows(), query_state::StdLog::log(), ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeApproxQuantileColumn(), ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeCountDistinctColumn(), ResultSet::parallelRowCount(), ResultSet::parallelTop(), and logger::Logger::stream().

                   {
  return g_query_id.load();
}

Here is the caller graph for this function:

void logger::set_once_fatal_func

(

FatalFunc

fatal_func

)

Definition at line 337 of file Logger.cpp.

Referenced by QueryRunner::QueryRunner::QueryRunner().

                                               {
  if (g_fatal_func.exchange(fatal_func)) {
    throw std::runtime_error(
        "logger::set_once_fatal_func() should not be called more than once.");
  }
}

Here is the caller graph for this function:

QidScopeGuard logger::set_thread_local_query_id

(

QueryId const

query_id

)

Definition at line 487 of file Logger.cpp.

References g_query_id.

Referenced by ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeApproxQuantileColumn(), ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeCountDistinctColumn(), ResultSet::parallelRowCount(), ResultSet::parallelTop(), and query_state::QueryState::setThreadLocalQueryId().

                                                                {
  QueryId expected = 0;
  return g_query_id.compare_exchange_strong(expected, query_id) ? QidScopeGuard(query_id)
                                                                : QidScopeGuard(0);
}

Here is the caller graph for this function:

void logger::shutdown

(

)

Definition at line 344 of file Logger.cpp.

References nvtx_helpers::shutdown().

Referenced by heartbeat(), EmbeddedDatabase::DBEngineImpl::reset(), TestProcessSignalHandler::shutdownSubsystemsAndExit(), startHeavyDBServer(), and logger::LogShutdown::~LogShutdown().

                {
  static std::once_flag logger_flag;
  std::call_once(logger_flag, []() {
    boost::log::core::get()->remove_all_sinks();
    nvtx_helpers::shutdown();
  });
}

Here is the call graph for this function:

Here is the caller graph for this function:

ThreadId logger::thread_id

(

)

Definition at line 820 of file Logger.cpp.

References g_thread_id.

Referenced by import_export::import_thread_delimited(), import_export::import_thread_shapefile(), import_export::Importer::importDelimited(), import_export::Importer::importGDALGeo(), import_export::Importer::importGDALRaster(), Executor::launchKernels(), foreign_storage::LazyParquetChunkLoader::loadRowGroups(), query_state::StdLog::log(), nvtx_helpers::name_current_thread(), StringDictionary::populate_string_array_ids(), Parser::InsertIntoTableAsSelectStmt::populateData(), PerfectJoinHashTable::reify(), OverlapsJoinHashTable::reifyImpl(), RangeJoinHashTable::reifyWithLayout(), BaselineJoinHashTable::reifyWithLayout(), and logger::Logger::stream().

                     {
  return g_thread_id;
}

Here is the caller graph for this function:

Variable Documentation

constexpr std::array<char const*, 3> logger::ChannelNames {"IR", "PTX", "ASM"}

Definition at line 90 of file Logger.h.

Referenced by operator<<(), and operator>>().

constexpr std::array<char, 3> logger::ChannelSymbols {'R', 'P', 'A'}

Definition at line 92 of file Logger.h.

bool logger::g_any_active_channels {false}

Definition at line 303 of file Logger.cpp.

Referenced by fast_logging_check(), and init().

DurationTreeMap logger::g_duration_tree_map

Definition at line 597 of file Logger.cpp.

Referenced by debug_timer_new_thread(), logAndEraseDurationTree(), newDuration(), and logger::EraseDurationTrees::operator()().

std::mutex logger::g_duration_tree_map_mutex

Definition at line 596 of file Logger.cpp.

Referenced by debug_timer_new_thread(), logAndEraseDurationTree(), and newDuration().

std::once_flag logger::g_fatal_func_flag

Definition at line 286 of file Logger.cpp.

Referenced by logger::Logger::~Logger().

boost::filesystem::path logger::g_log_dir_path

static

Definition at line 306 of file Logger.cpp.

Referenced by get_log_dir_path(), and init().

Severity logger::g_min_active_severity {Severity::FATAL}

Definition at line 304 of file Logger.cpp.

Referenced by fast_logging_check(), and init().

std::atomic<ThreadId> logger::g_next_thread_id {0}

Definition at line 598 of file Logger.cpp.

Referenced by debug_timer_new_thread().

thread_local std::atomic<QueryId> logger::g_query_id {0}

Definition at line 471 of file Logger.cpp.

Referenced by query_id(), set_thread_local_query_id(), and logger::QidScopeGuard::~QidScopeGuard().

thread_local ThreadId logger::g_thread_id = g_next_thread_id++

Definition at line 599 of file Logger.cpp.

Referenced by debug_timer_new_thread(), logAndEraseDurationTree(), newDuration(), and thread_id().

constexpr std::array<char const*, 8> logger::SeverityNames

Initial value:

{"DEBUG4",
                                                   "DEBUG3",
                                                   "DEBUG2",
                                                   "DEBUG1",
                                                   "INFO",
                                                   "WARNING",
                                                   "ERROR",
                                                   "FATAL"}

Definition at line 112 of file Logger.h.

constexpr std::array<char, 8> logger::SeveritySymbols {'4', '3', '2', '1', 'I', 'W', 'E', 'F'}

Definition at line 121 of file Logger.h.