OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
foreign_storage::LogFileBufferParser Class Reference

#include <LogFileBufferParser.h>

+ Inheritance diagram for foreign_storage::LogFileBufferParser:
+ Collaboration diagram for foreign_storage::LogFileBufferParser:

Public Member Functions

 LogFileBufferParser (const ForeignTable *foreign_table, int32_t db_id)
 
- Public Member Functions inherited from foreign_storage::RegexFileBufferParser
 RegexFileBufferParser (const ForeignTable *foreign_table)
 
ParseBufferResult parseBuffer (ParseBufferRequest &request, bool convert_data_blocks, bool columns_are_pre_filtered=false, bool skip_dict_encoding=false) const override
 
import_export::CopyParams validateAndGetCopyParams (const ForeignTable *foreign_table) const override
 
size_t findRowEndPosition (size_t &alloc_size, std::unique_ptr< char[]> &buffer, size_t &buffer_size, const import_export::CopyParams &copy_params, const size_t buffer_first_row_index, unsigned int &num_rows_in_buffer, FileReader *file_reader) const override
 
void validateFiles (const FileReader *file_reader, const ForeignTable *foreign_table) const override
 

Protected Member Functions

bool regexMatchColumns (const std::string &row_str, const boost::regex &line_regex, size_t logical_column_count, std::vector< std::string > &parsed_columns_str, std::vector< std::string_view > &parsed_columns_sv, const std::string &file_path) const override
 
bool shouldRemoveNonMatches () const override
 
bool shouldTruncateStringValues () const override
 

Private Attributes

const ForeignTableforeign_table_
 
const int32_t db_id_
 

Additional Inherited Members

- Static Public Member Functions inherited from foreign_storage::RegexFileBufferParser
static void setMaxBufferResize (size_t max_buffer_resize)
 
- Static Public Member Functions inherited from foreign_storage::TextFileBufferParser
static std::map< int,
DataBlockPtr
convertImportBuffersToDataBlocks (const std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const bool skip_dict_encoding=false)
 
static bool isCoordinateScalar (const std::string_view datum)
 
static void processGeoColumn (std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, size_t &col_idx, const import_export::CopyParams &copy_params, std::list< const ColumnDescriptor * >::iterator &cd_it, std::vector< std::string_view > &row, size_t &import_idx, bool is_null, size_t first_row_index, size_t row_index_plus_one, std::shared_ptr< Catalog_Namespace::Catalog > catalog)
 
static void fillRejectedRowWithInvalidData (const std::list< const ColumnDescriptor * > &columns, std::list< const ColumnDescriptor * >::iterator &cd_it, const size_t col_idx, ParseBufferRequest &request)
 
static bool isNullDatum (const std::string_view datum, const ColumnDescriptor *column, const std::string &null_indicator)
 
- Static Public Attributes inherited from foreign_storage::RegexFileBufferParser
static const std::string LINE_REGEX_KEY = "LINE_REGEX"
 
static const std::string LINE_START_REGEX_KEY = "LINE_START_REGEX"
 
static const std::string HEADER_KEY = "HEADER"
 
- Static Public Attributes inherited from foreign_storage::TextFileBufferParser
static const std::string BUFFER_SIZE_KEY = "BUFFER_SIZE"
 

Detailed Description

Definition at line 22 of file LogFileBufferParser.h.

Constructor & Destructor Documentation

foreign_storage::LogFileBufferParser::LogFileBufferParser ( const ForeignTable foreign_table,
int32_t  db_id 
)

Definition at line 126 of file LogFileBufferParser.cpp.

127  : RegexFileBufferParser(foreign_table)
128  , foreign_table_(foreign_table)
129  , db_id_(db_id) {}
RegexFileBufferParser(const ForeignTable *foreign_table)

Member Function Documentation

bool foreign_storage::LogFileBufferParser::regexMatchColumns ( const std::string &  row_str,
const boost::regex &  line_regex,
size_t  logical_column_count,
std::vector< std::string > &  parsed_columns_str,
std::vector< std::string_view > &  parsed_columns_sv,
const std::string &  file_path 
) const
overrideprotectedvirtual

Reimplemented from foreign_storage::RegexFileBufferParser.

Definition at line 131 of file LogFileBufferParser.cpp.

References foreign_storage::anonymous_namespace{LogFileBufferParser.cpp}::add_column_value(), foreign_storage::anonymous_namespace{LogFileBufferParser.cpp}::add_nonce_values(), CHECK, CHECK_EQ, CHECK_GT, foreign_storage::anonymous_namespace{LogFileBufferParser.cpp}::create_map_from_arrays(), db_id_, foreign_table_, foreign_storage::anonymous_namespace{LogFileBufferParser.cpp}::get_node_name(), foreign_storage::RegexFileBufferParser::regexMatchColumns(), TableDescriptor::tableId, and TableDescriptor::tableName.

137  {
138  CHECK(parsed_columns_str.empty());
139  CHECK(parsed_columns_sv.empty());
141  if (foreign_table_->tableName == "request_logs") {
142  boost::smatch match;
143  bool set_all_nulls{false};
144  if (boost::regex_match(row_str, match, line_regex)) {
145  auto matched_column_count = match.size() - 1;
146  // Last 2 matched columns are associative arrays.
147  CHECK_GT(matched_column_count, size_t(2));
148  for (size_t i = 1; i < match.size() - 2; i++) {
149  add_column_value(parsed_columns_str, parsed_columns_sv, match[i].str());
150  }
151  // Special handling for associative arrays.
152  auto values_map = create_map_from_arrays(match[matched_column_count - 1].str(),
153  match[matched_column_count].str());
154  static const std::array<std::string, 5> keys{
155  "query_str", "client", "nonce", "execution_time_ms", "total_time_ms"};
156  CHECK_EQ(logical_column_count, matched_column_count + keys.size());
157  for (const auto& key : keys) {
158  auto it = values_map.find(key);
159  if (it == values_map.end()) {
160  if (key == "nonce") {
161  // Null dashboard id, dashboard name, and chart id.
162  add_column_value(parsed_columns_str, parsed_columns_sv, "", 3);
163  } else {
164  // Add null value for missing entry.
165  add_column_value(parsed_columns_str, parsed_columns_sv, "");
166  }
167  } else {
168  if (key == "nonce") {
169  add_nonce_values(parsed_columns_str,
170  parsed_columns_sv,
171  it->second,
173  db_id_);
174  } else {
175  add_column_value(parsed_columns_str, parsed_columns_sv, it->second);
176  }
177  }
178  }
179  CHECK_EQ(parsed_columns_str.size(), parsed_columns_sv.size());
180  } else {
181  parsed_columns_str.clear();
182  parsed_columns_sv =
183  std::vector<std::string_view>(logical_column_count, std::string_view{});
184  set_all_nulls = true;
185  }
186  CHECK_EQ(parsed_columns_sv.size(), logical_column_count) << "In row: " << row_str;
187  return set_all_nulls;
188  } else {
189  // Add fixed value for the server_logs table "node" column.
190  if (foreign_table_->tableName == "server_logs") {
191  add_column_value(parsed_columns_str, parsed_columns_sv, get_node_name());
192  }
194  line_regex,
195  logical_column_count,
196  parsed_columns_str,
197  parsed_columns_sv,
198  file_path);
199  }
200 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
std::string tableName
void add_nonce_values(std::vector< std::string > &parsed_columns_str, std::vector< std::string_view > &parsed_columns_sv, const std::string &nonce, int32_t table_id, int32_t db_id)
#define CHECK_GT(x, y)
Definition: Logger.h:305
virtual bool regexMatchColumns(const std::string &row_str, const boost::regex &line_regex, size_t logical_column_count, std::vector< std::string > &parsed_columns_str, std::vector< std::string_view > &parsed_columns_sv, const std::string &file_path) const
std::map< std::string, std::string > create_map_from_arrays(const std::string &keys_array, const std::string &values_array)
#define CHECK(condition)
Definition: Logger.h:291
void add_column_value(std::vector< std::string > &parsed_columns_str, std::vector< std::string_view > &parsed_columns_sv, const std::string &value, size_t count=1)

+ Here is the call graph for this function:

bool foreign_storage::LogFileBufferParser::shouldRemoveNonMatches ( ) const
overrideprotectedvirtual

Reimplemented from foreign_storage::RegexFileBufferParser.

Definition at line 202 of file LogFileBufferParser.cpp.

202  {
203  return true;
204 }
bool foreign_storage::LogFileBufferParser::shouldTruncateStringValues ( ) const
overrideprotectedvirtual

Reimplemented from foreign_storage::RegexFileBufferParser.

Definition at line 206 of file LogFileBufferParser.cpp.

206  {
207  return true;
208 }

Member Data Documentation

const int32_t foreign_storage::LogFileBufferParser::db_id_
private

Definition at line 40 of file LogFileBufferParser.h.

Referenced by regexMatchColumns().

const ForeignTable* foreign_storage::LogFileBufferParser::foreign_table_
private

Definition at line 39 of file LogFileBufferParser.h.

Referenced by regexMatchColumns().


The documentation for this class was generated from the following files: